Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
text_raw = np.hstack([train.item_description.str.lower() , train.name.str.lower() ])
 tok = Tokenizer()
tok.fit_on_texts(text_raw)
 train['seq_item_description'] = tok.texts_to_sequences(train.item_description.str.lower())
 test['seq_item_description'] = tok.texts_to_sequences(test.item_description.str.lower())
 train['seq_name'] = tok.texts_to_sequences(train.name.str.lower())
 test['seq_name'] = tok.texts_to_sequences(test.name.str.lower() )<prepare_output>
def get_title(data_frame):
name_data = data_frame["Name"]
data_frame["Title"] = [name.split(", ", 1)[1].split(".", 1)[0] for name in name_data]
titles = []
for title in data_frame["Title"]:
if title not in titles:
titles.append(title)
 return data_frame, titles
train, titles = get_title(train)
 print(titles )
max_name_seq = np.max([np.max(train.seq_name.apply(lambda x: len(x))),
np.max(test.seq_name.apply(lambda x: len(x)))])
 max_seq_item_des = np.max([np.max(train.seq_item_description.apply(lambda x: len(x))),
np.max(test.seq_item_description.apply(lambda x: len(x)))] )<define_variables>
def title2int(data):
data["Title"].replace(["Major", "Capt", "Sir", "Dr", "Don", "Mlle", "Mme", "Ms", "Dona", "Lady", "the Countess", "Jonkheer", "Col", "Rev"],
["Mr", "Mr", "Mr", "Mr", "Mr", "Miss", "Miss", "Miss", "Mrs", "Mrs", "Mrs", "Other", "Other", "Other"], inplace = True)
 data["Title"].replace(["Mr", "Miss", "Mrs", "Master", "Other"], [0, 1, 2, 3, 4], inplace = True)
 return data
train = title2int(train )
MAX_NAME_SEQ = 17
MAX_ITEM_DESCRIPTION = 70
MAX_TEXT = np.max([np.max(train.seq_name.max()), np.max(test.seq_name.max()),
np.max(train.seq_item_description.max()), np.max(test.seq_item_description.max())])+ 2
MAX_BRAND_NAME = np.max([train.brand_name.max() , test.brand_name.max() ])+ 1
MAX_CATEGORY_NAME = np.max([train.category_name.max() , test.category_name.max() ])+ 1
MAX_CONDITION = np.max([train.item_condition_id.max() , test.item_condition_id.max() ])+ 1<normalization>
train.groupby("Title")["Age"].mean()
train['target'] = np.log(train.price + 1)
 target_scaler = MinMaxScaler(feature_range=(-1, 1))
train['target'] = target_scaler.fit_transform(train['target'].reshape(-1, 1))
dtrain, dvalid = train_test_split(train, random_state=42, train_size = 0.95 )<prepare_x_and_y>
train.groupby("Pclass")["Fare"].mean()
def get_keras_data(data):
X = {
'name': pad_sequences(data.seq_name, maxlen=MAX_NAME_SEQ),
'item_desc': pad_sequences(data.seq_item_description, maxlen=MAX_ITEM_DESCRIPTION),
'brand_name': np.array(data.brand_name),
'category_name': np.array(data.category_name),
'item_condition': np.array(data.item_condition_id),
'num_vars': np.array(data[['shipping']])
 }
return X
X_train = get_keras_data(dtrain)
 X_valid = get_keras_data(dvalid)
 X_test = get_keras_data(test )<compute_train_metric>
def fareG2int(data):
data["Fare_group"] = "NaN"
data.loc[data["Fare"] < 10, "Fare_group"] = 2
data.loc[(data["Fare"] >= 10)&(data["Fare"] < 65), "Fare_group"] = 2
data.loc[data["Fare"] >= 65, "Fare_group"] = 1
return data
train = fareG2int(train )
def def_get_callback(filepath, patience=2):
es = EarlyStopping('val_loss', partience=partience, mode='min')
 msave = ModelCheckpoint(filepath, save_best_only=True)
 return [es, msave]
def rmsle_cust(y_true, y_pred):
first_log = backend.log(backend.clip(y_pred, backend.epsilon() , None)+ 1)
 second_log = backend.log(backend.clip(y_true, backend.epsilon() , None)+ 1)
 return backend.sqrt(backend.mean(backend.square(first_log - second_log), axis=-1))
def get_model() :
name = Input(shape=[X_train['name'].shape[1]], name='name')
 item_desc = Input(shape=[X_train['item_desc'].shape[1]], name='item_desc')
 brand_name = Input(shape=[1], name='brand_name')
 category_name = Input(shape=[1], name='category_name')
 item_condition = Input(shape=[1], name='item_condition')
 num_vars = Input(shape=[X_train['num_vars'].shape[1]], name='num_vars')
 emb_name = Embedding(MAX_TEXT, 30 )(name)
 emb_item_desc = Embedding(MAX_TEXT, 30 )(item_desc)
 emb_brand_name = Embedding(MAX_BRAND_NAME, 10 )(brand_name)
 emb_category_name = Embedding(MAX_CATEGORY_NAME, 10 )(category_name)
 emb_item_condition = Embedding(MAX_CONDITION, 5 )(item_condition)
 rnn_layer1 = GRU(40 )(emb_item_desc)
 rnn_layer2 = GRU(20 )(emb_name)
 rnn_layer3 = GRU(10 )(emb_brand_name)
 rnn_layer4 = GRU(10 )(emb_category_name)
 main_l = concatenate([Flatten()(emb_brand_name), Flatten()(emb_category_name), Flatten()(emb_item_condition), rnn_layer1, rnn_layer2, rnn_layer3, rnn_layer4, num_vars])
 main_l = Dropout(0.1 )(Dense(512, activation='relu' )(main_l))
main_l = Dropout(0.1 )(Dense(64, activation='relu' )(main_l))
output = Dense(1, activation='linear' )(main_l)
 model = Model([name, item_desc, brand_name, category_name, item_condition, num_vars], output)
 model.compile(loss='mse', optimizer='adam', metrics=['mae', rmsle_cust])
 return model
model = get_model()<train_model>
train["Embarked"] = train["Embarked"].fillna("S" )
batch_size = 20000
epochs = 5
model.fit(X_train, dtrain.target, epochs=epochs, batch_size=batch_size, validation_data=(X_valid, dvalid.target), verbose=1 )<predict_on_test>
train["Embarked"].replace(["S", "Q", "C"], [0, 1, 2], inplace = True )
val_preds = model.predict(X_valid)
 val_preds = target_scaler.inverse_transform(val_preds)
 val_preds = np.exp(val_preds)+ 1
y_true = np.array(dvalid.price.values)
 y_pred = val_preds[:,0]
rmsle(y_true, y_pred )<predict_on_test>
def cab2int(data):
data.loc[data["Cabin"] == "Known", 'Cabin'] = 1
data.loc[data["Cabin"] == "Unknown", 'Cabin'] = 0
return data
train = cab2int(train )
preds_train = model.predict(X_train, batch_size=batch_size)
 preds_train = target_scaler.inverse_transform(preds_train)
 preds_train = np.exp(preds_train)- 1
dtrain['price_rnn'] = preds_train
preds_valid = model.predict(X_valid, batch_size=batch_size)
 preds_valid = target_scaler.inverse_transform(preds_valid)
 preds_valid = np.exp(preds_valid)- 1
dvalid['price_rnn'] = preds_valid<import_modules>
train["Sex"].replace(["male", "female"], [0, 1], inplace = True )
from xgboost import XGBRegressor<prepare_x_and_y>
def assign_missing_ages(data_frame, features):
age_data = data_frame[features]
known_ages = age_data[age_data.Age.notnull() ].as_matrix()
unknown_ages = age_data[age_data.Age.isnull() ].as_matrix()
target = known_ages[:, 0]
eigen_val = known_ages[:, 1:]
rfr = RandomForestRegressor(random_state = 0, n_estimators = 2000, n_jobs = -1)
 rfr.fit(eigen_val, target)
 Age_predictions = rfr.predict(unknown_ages[:, 1::])
 data_frame.loc[(data_frame.Age.isnull()), "Age"] = Age_predictions
return data_frame, rfr
age_features = ["Age", "Sex", "SibSp", "Parch", "Pclass"]
train, rfr = assign_missing_ages(train, age_features)
 train["Age"] = train["Age"].astype(int)
 train.head(10 )
X_train_xgb = dtrain[['item_condition_id', 'category_name', 'brand_name', 'shipping', 'price_rnn']]
y_train_xgb = dtrain.price
X_valid_xgb = dvalid[['item_condition_id', 'category_name', 'brand_name', 'shipping', 'price_rnn']]
y_valid_xgb = dvalid.price<train_model>
train.groupby("Title")["Age"].mean()
model_xgb = XGBRegressor(booster='gbtree', max_depth=13, n_estimators=250, eta=0.05, reg_lambda=4, reg_alpha=2)
 model_xgb.fit(X_train_xgb, y_train_xgb)
 pred = model_xgb.predict(X_valid_xgb)
 print(rmsle(y_valid_xgb, pred))<predict_on_test>
def ageG2int(data):
data["Age_group"] = "NaN"
data.loc[data["Age"] <= 16, "Age_group"] = 0
data.loc[(data["Age"] > 16)&(data["Age"] <= 32), "Age_group"] = 1
data.loc[(data["Age"] > 32)&(data["Age"] <= 48), "Age_group"] = 3
data.loc[(data["Age"] > 48)&(data["Age"] <= 64), "Age_group"] = 4
data.loc[data["Age"] > 64, "Age_group"] = 5
return data
train = ageG2int(train )
preds_test = model.predict(X_test, batch_size=batch_size)
 preds_test = target_scaler.inverse_transform(preds_test)
 preds_test = np.exp(preds_test)- 1
test['price_rnn'] = preds_test<save_to_csv>
def child2int(data):
data["Child"] = "NaN"
data.loc[data["Age"] <= 18, "Child"] = 0
data.loc[data["Age"] > 18, "Child"] = 1
return data
train = child2int(train )
test_xgb = test[['item_condition_id', 'category_name', 'brand_name', 'shipping', 'price_rnn']]
preds = model_xgb.predict(test_xgb)
 index = pd.Series(np.arange(len(test_xgb)) , name='test_id')
 price = pd.Series(preds, name='price')
 submission = pd.concat([index, price], axis=1)
 submission.to_csv('submission.csv', index=False )<install_modules>
train["FamSize"] = train["SibSp"] + train["Parch"] + 1
!pip install -q efficientnet_pytorch > /dev/null<set_options>
def famG2int(data):
data["Fam_group"] = "NaN"
data.loc[data["FamSize"] == 1, "Fam_group"] = 0
data.loc[data["FamSize"] > 1, "Fam_group"] = 1
return data
train = famG2int(train )
SEED = 512
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<load_from_csv>
train_one = train[:]
<normalization>
test["Embarked"].replace(["S", "Q", "C"], [0, 1, 2], inplace = True)
 test["Fare"] = test["Fare"].fillna(test["Fare"].median())
 test, test_titles = get_title(test)
 test = title2int(test)
 test["Sex"].replace(["male", "female"], [0, 1], inplace = True)
 test = Cabin_type(test)
 test = cab2int(test)
 test = fareG2int(test)
 temp_test = test[age_features]
test_unknown_ages = temp_test[test["Age"].isnull() ].as_matrix()
test_Age_predictions = rfr.predict(test_unknown_ages[:, 1:])
 test.loc[(test["Age"].isnull()), "Age"] = test_Age_predictions
test["Age"] = test["Age"].astype(int)
 test = ageG2int(test)
 test = child2int(test)
 test["FamSize"] = test["SibSp"] + test["Parch"] + 1
test = famG2int(test)
 test_one = test[:]
test_columns_titles = ["PassengerId", "Pclass", "Title", "Sex", "Child", "Fam_group", "Fare", "Cabin", "Embarked"]
test_one = test_one[test_columns_titles]
test_one.info()
test_one.head()
def get_train_transforms() :
return A.Compose([
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0)
 def get_valid_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<categorify>
def my_models(model, X_train, Y_train, X_test, Y_test):
my_model = model.fit(X_train, Y_train)
 print(my_model.feature_importances_)
 print(my_model.score(X_train, Y_train))
model_prediction = my_model.predict(X_test)
 acc = metrics.accuracy_score(model_prediction, Y_test)
 return acc, my_model
DATA_ROOT_PATH = '.. /input/alaska2-image-steganalysis'
def onehot(size, target):
vec = torch.zeros(size, dtype=torch.float32)
 vec[target] = 1.
 return vec
class DatasetRetriever(Dataset):
def __init__(self, kinds, image_names, labels, transforms=None):
super().__init__()
self.kinds = kinds
self.image_names = image_names
self.labels = labels
self.transforms = transforms
def __getitem__(self, index: int):
kind, image_name, label = self.kinds[index], self.image_names[index], self.labels[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{kind}/{image_name}', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
target = onehot(4, label)
 return image, target
def __len__(self)-> int:
return self.image_names.shape[0]
def get_labels(self):
return list(self.labels )<define_variables>
final_features = ["Pclass", "Title", "Sex", "Child", "Fam_group", "Fare", "Cabin", "Embarked"]
final_data = train_one[["Survived"] + final_features]
training, testing = train_test_split(final_data, test_size = 0.3, random_state = 0, stratify = final_data["Survived"])
 X_train = training[training.columns[1:]]
Y_train = training[training.columns[:1]]
X_test = testing[testing.columns[1:]]
Y_test = testing[testing.columns[:1]]
fold_number = 0
train_dataset = DatasetRetriever(
 kinds=dataset[dataset['fold'] != fold_number].kind.values,
image_names=dataset[dataset['fold'] != fold_number].image_name.values,
labels=dataset[dataset['fold'] != fold_number].label.values,
transforms=get_train_transforms() , 
)
 validation_dataset = DatasetRetriever(
 kinds=dataset[dataset['fold'] == fold_number].kind.values,
image_names=dataset[dataset['fold'] == fold_number].image_name.values,
labels=dataset[dataset['fold'] == fold_number].label.values,
transforms=get_valid_transforms() ,
)<compute_train_metric>
tree_model = tree.DecisionTreeClassifier(max_depth = 8, max_leaf_nodes = 7, min_samples_leaf = 10, random_state = 0)
 forest_model = RandomForestClassifier(max_depth = 8, max_leaf_nodes = 9, n_estimators = 300, random_state = 0)
 gradboost_model = GradientBoostingClassifier(learning_rate = 0.01, max_depth = 7,
max_features = 1.0, n_estimators = 200, subsample = 1.0, random_state = 0 )
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def alaska_weighted_auc(y_true, y_valid):

 tpr_thresholds = [0.0, 0.4, 1.0]
weights = [2, 1]
fpr, tpr, thresholds = metrics.roc_curve(y_true, y_valid, pos_label=1)
 areas = np.array(tpr_thresholds[1:])- np.array(tpr_thresholds[:-1])
 normalization = np.dot(areas, weights)
 competition_metric = 0
for idx, weight in enumerate(weights):
y_min = tpr_thresholds[idx]
y_max = tpr_thresholds[idx + 1]
mask =(y_min < tpr)&(tpr < y_max)
 x_padding = np.linspace(fpr[mask][-1], 1, 100)
 x = np.concatenate([fpr[mask], x_padding])
 y = np.concatenate([tpr[mask], [y_max] * len(x_padding)])
 y = y - y_min
score = metrics.auc(x, y)
 submetric = score * weight
best_subscore =(y_max - y_min)* weight
competition_metric += submetric
return competition_metric / normalization
class RocAucMeter(object):
def __init__(self):
self.reset()
def reset(self):
self.y_true = np.array([0,1])
 self.y_pred = np.array([0.5,0.5])
 self.score = 0
def update(self, y_true, y_pred):
y_true = y_true.cpu().numpy().argmax(axis=1 ).clip(min=0, max=1 ).astype(int)
 y_pred = 1 - nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,0]
self.y_true = np.hstack(( self.y_true, y_true))
self.y_pred = np.hstack(( self.y_pred, y_pred))
self.score = alaska_weighted_auc(self.y_true, self.y_pred)
 @property
def avg(self):
return self.score<normalization>
tree_acc, my_tree = my_models(tree_model, X_train, Y_train, X_test, Y_test)
 print("The accuracy of Decision Tree is", tree_acc)
 forest_acc, my_forest = my_models(forest_model, X_train, Y_train, X_test, Y_test)
 print("The accuracy of Random Forest is", forest_acc)
 gradboost_acc, my_gradboost = my_models(gradboost_model, X_train, Y_train, X_test, Y_test)
 print("The accuracy of Gradient Boosting is", gradboost_acc )
<init_hyperparams><EOS> 
final_test = test_one[final_features]
tree_prediction = my_tree.predict(final_test)
 forest_prediction = my_forest.predict(final_test)
 gradboost_prediction = my_gradboost.predict(final_test)
 test_cp1 = test_one[:]
test_cp2 = test_one[:]
test_cp3 = test_one[:]
headers = ["PassengerId", "Survived"]
test_cp1["Survived"] = tree_prediction
tree_prediction = pd.DataFrame(test_cp1, columns = headers)
 tree_prediction.to_csv("tree_prediction.csv", index = False)
 test_cp2["Survived"] = forest_prediction
forest_prediction = pd.DataFrame(test_cp2, columns = headers)
 forest_prediction.to_csv("forest_prediction.csv", index = False)
 test_cp3["Survived"] = gradboost_prediction
gradboost_prediction = pd.DataFrame(test_cp3, columns = headers)
 gradboost_prediction.to_csv("gradboost_prediction.csv", index = False )