Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<split>
%matplotlib inline
%config IPCompleter.greedy=True
data = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 print('Data Set',data.shape)
 print('Test Set',test.shape )
y = X_train_non_null.AdoptionSpeed
X_train_cat_y = X_train_cat.copy()
X_train_cat_y["AdoptionSpeed"] = y
kfold = GroupKFold(n_splits=5)
 X_train_cat_encoded = np.zeros(( X_train_cat.shape[0], len(cat_features)* 2))
X_test_cat_encoded = np.zeros(( X_test_cat.shape[0], len(cat_features)* 2))
for trn_idx, val_idx in kfold.split(X_train_cat, y, train_pet_ids):
X_trn, X_val = X_train_cat_y.loc[trn_idx, :], X_train_cat_y.loc[val_idx, :]
for j, c in enumerate(cat_features):
X_trn_enc = X_trn.groupby(c ).agg({
"AdoptionSpeed": ["mean", "std"]
})
 cte_columns = [f"{c}_{x}" for x in ["mean", "std"]]
X_trn_enc.columns = cte_columns
X_temp = np.zeros(( X_test_cat.shape[0], 2))
X_temp_df = pd.DataFrame(data=X_temp, columns=cte_columns)
 for x in X_trn_enc.columns:
X_val[x] = X_val[c].map(X_trn_enc[x])
 X_temp_df[x] = X_test_cat[c].map(X_trn_enc[x] ).reset_index(drop=True)
 X_train_cat_encoded[val_idx, 2 * j:2 *(j + 1)] = X_val[X_trn_enc.columns].values
X_test_cat_encoded[:, 2 * j: 2 *(j + 1)] += X_temp_df.values / 5<prepare_output>
print('Data Set
',data.isnull().sum())
 print('---------------')
 print('Test Set
',test.isnull().sum() )
columns = [f"{c}_{x}" for c in cat_features for x in ["mean", "std"]]
X_train_cat_encoded_df = pd.DataFrame(data=X_train_cat_encoded, columns=columns)
 X_test_cat_encoded_df = pd.DataFrame(data=X_test_cat_encoded, columns=columns)
 X_test_cat_encoded_df.head()<prepare_x_and_y>
sns.set()
X_train_num = X_train_non_null.drop(cat_features + ["AdoptionSpeed"], axis=1)
 X_test_num = X_test_non_null.drop(cat_features, axis=1)
 target = X_train_non_null["AdoptionSpeed"]<concatenate>
data.Sex.replace(['male','female'],[1,0],inplace = True)
 test.Sex.replace(['male','female'],[1,0],inplace = True)
 data.head()
X_train_num = pd.concat([X_train_num, X_train_cat_encoded_df], axis=1)
 X_test_cat_encoded_df.index = X_test_num.index
X_test_num = pd.concat([X_test_num, X_test_cat_encoded_df], axis=1 )<normalization>
data['Title'] = data['Name'].str.extract('([A-Za-z]+)\.', expand=False)
 data['Title'].value_counts()
test['Title'] = test['Name'].str.extract('([A-Za-z]+)\.', expand=False)
 test['Title'].value_counts()
X_all_num = pd.concat([X_train_num, X_test_num], axis=0)
 ss = StandardScaler()
ss.fit(X_all_num)
 X_train_ss = ss.transform(X_train_num)
 X_test_ss = ss.transform(X_test_num )<feature_engineering>
data.drop('Name',axis = 1,inplace = True)
 test.drop('Name',axis = 1,inplace = True)
 data.drop('Ticket',axis = 1,inplace = True)
 test.drop('Ticket',axis = 1,inplace = True )
X_train_ss[np.isnan(X_train_ss)] = 0.0
X_test_ss[np.isnan(X_test_ss)] = 0.0<count_unique_values>
data['Age'].fillna(data.groupby('Title')['Age'].transform('median'),inplace = True)
 test['Age'].fillna(data.groupby('Title')['Age'].transform('median'),inplace = True )
cat_cat = pd.concat([X_train_cat, X_test_cat])
 n_breed1 = cat_cat["Breed1"].nunique()
n_breed2 = cat_cat["Breed2"].nunique()
n_langs = cat_cat["language"].nunique()
n_color1 = cat_cat["Color1"].nunique()
n_color2 = cat_cat["Color2"].nunique()
n_color3 = cat_cat["Color3"].nunique()<categorify>
data['Fare'].fillna(data.groupby('Pclass')['Fare'].transform('median'),inplace = True)
 test['Fare'].fillna(data.groupby('Pclass')['Fare'].transform('median'),inplace = True )
for c in X_train_cat.columns:
le = LabelEncoder()
le.fit(cat_cat[c])
 X_train_cat[c] = le.transform(X_train_cat[c])
 X_test_cat[c] = le.transform(X_test_cat[c] )<train_on_grid>
data['Embarked'].fillna(3,inplace = True)
 test['Embarked'].fillna(3,inplace = True )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y, initial_coef=[]):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = initial_coef
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):
assert len(rater_a)== len(rater_b)
 if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_rating = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_rating)] for j in range(num_rating)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):
if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):
rater_a = y
rater_b = y_pred
min_rating = None
max_rating = None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert len(rater_a)== len(rater_b)
 min_rating = min(min(rater_a), min(rater_b))
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b, min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(
 hist_rater_a[i] * hist_rater_b[j])/ num_scored_items
d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<create_dataframe>
data.isnull().sum()
class PetDataset(data.Dataset):
def __init__(self,
pet_ids,
cat_features,
num_features,
labels,
root_dir,
subset=False,
transform=None):
self.pet_ids = pet_ids
self.cat_features = cat_features
self.num_features = num_features
if labels is not None:
self.labels = labels
else:
self.labels = None
self.root_dir = root_dir
self.transform = transform
def __len__(self):
return len(self.pet_ids)
 def __getitem__(self, idx):
img_name = f"{self.pet_ids[idx]}-1.jpg"
fullname = self.root_dir / Path(img_name)
 try:
image = Image.open(fullname ).convert("RGB")
 except FileNotFoundError:
image = np.zeros(( 3, 224, 224), dtype=np.uint8 ).transpose(1, 2, 0)
 image = Image.fromarray(np.uint8(image))
cat_feature = self.cat_features[idx]
num_feature = self.num_features[idx]
if self.transform:
image = self.transform(image)
 if self.labels is not None:
label = self.labels[idx]
return [image, cat_feature, num_feature, label]
else:
return [image, cat_feature, num_feature]<normalization>
data.isnull().sum()
normalize = transforms.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225] 
)
 ds_trans = transforms.Compose([transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ToTensor() ,
normalize] )<init_hyperparams>
data['Cabin']= data['Cabin'].str[:1]
data['Cabin']
def get_n_params(model):
pp=0
for p in list(model.parameters()):
nn=1
for s in list(p.size()):
nn = nn*s
pp += nn
return pp
def seed_torch(seed=1029):
random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
def run_xgb(params,
X,
y,
X_test,
resc,
n_splits=10,
num_rounds=60000,
early_stop=500,
verbose_eval=1000):
fold = GroupKFold(n_splits=n_splits)
 oof_train = np.zeros(( X.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
for i,(trn_index, val_index)in enumerate(fold.split(X, y, train_pet_ids)) :
X_tr = X.iloc[trn_index, :]
X_val = X.iloc[val_index, :]
y_tr = y[trn_index]
y_val = y[val_index]
d_train = xgb.DMatrix(
 data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(
 data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, "train"),(d_valid, "valid")]
model = xgb.train(
 params=params,
dtrain=d_train,
num_boost_round=num_rounds,
evals=watchlist,
early_stopping_rounds=early_stop,
verbose_eval=verbose_eval)
 valid_pred = model.predict(
 xgb.DMatrix(X_val, feature_names=X_val.columns),
ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(
 xgb.DMatrix(X_test, feature_names=X_test.columns),
ntree_limit=model.best_ntree_limit)
 oof_train[val_index] = valid_pred
oof_test[:, i] = test_pred
return model, oof_train, oof_test
class Trainer:
def __init__(self,
model,
resc,
n_splits=5,
seed=42,
device="cuda:0",
train_batch=16,
val_batch=32,
kwargs={}):
self.model = model
self.resc = resc
self.n_splits = n_splits
self.seed = seed
self.device = device
self.train_batch = train_batch
self.val_batch = val_batch
self.kwargs = kwargs
self.fold = GroupKFold(
 n_splits=n_splits)
 self.best_score = None
self.tag = dt.now().strftime("%Y-%m-%d-%H-%M-%S")
 self.loss_fn = nn.MSELoss(reduction="mean" ).to(self.device)
 path = Path(f"bin/{self.tag}")
 path.mkdir(exist_ok=True, parents=True)
 self.path = path
def fit(self, pet_ids, cat_feats, num_feats, answer, n_epochs=30):
self.train_preds = np.zeros(( train.shape[0]))
answer = answer.values
cat_feats = cat_feats.values
for i,(trn_idx, val_idx)in enumerate(self.fold.split(pet_ids, answer, self.resc)) :
self.fold_num = i
print(f"Fold: {i+1}")
 pid_train, pid_val = pet_ids[trn_idx], pet_ids[val_idx]
cat_train, cat_val = cat_feats[trn_idx], cat_feats[val_idx]
num_train, num_val = num_feats[trn_idx], num_feats[val_idx]
y_train, y_val = answer[trn_idx] / 4, answer[val_idx] / 4
valid_preds = self._fit(pid_train,
cat_train,
num_train,
y_train,
n_epochs,
pid_val,
cat_val,
num_val,
y_val 
)
 self.train_preds[val_idx] = valid_preds
def _fit(self, pid, cat, num, y, n_epochs, pid_val, cat_val, num_val, y_val):
seed_torch(self.seed)
 cat_tensor = torch.tensor(cat, dtype=torch.long ).to(self.device)
 num_tensor = torch.tensor(num, dtype=torch.float32 ).to(self.device)
 y_tensor = torch.tensor(y[:, np.newaxis], dtype=torch.float32 ).to(self.device)
 train = PetDataset(pid,
cat_tensor,
num_tensor,
y_tensor,
".. /input/petfinder-adoption-prediction/train_images/",
transform=ds_trans)
 train_loader = data.DataLoader(train,
batch_size=self.train_batch, shuffle=True)
 cat_eval = torch.tensor(cat_val, dtype=torch.long ).to(self.device)
 num_eval = torch.tensor(num_val, dtype=torch.float32 ).to(self.device)
 y_eval = torch.tensor(y_val[:, np.newaxis], dtype=torch.float32 ).to(self.device)
 eval_ = PetDataset(pid_val,
cat_eval,
num_eval,
y_eval,
".. /input/petfinder-adoption-prediction/train_images/",
transform=ds_trans)
 eval_loader = data.DataLoader(eval_,
batch_size=self.val_batch, shuffle=False)
 model = self.model(**self.kwargs)
 model = model.to(self.device)
 optimizer = optim.Adam(model.parameters())
 best_score = np.inf
for epoch in range(n_epochs):
model.train()
avg_loss = 0.
 for i_batch, c_batch, n_batch, y_batch in tqdm(train_loader):
i_batch = i_batch.to(self.device)
 y_pred = model(i_batch, c_batch, n_batch)
 loss = self.loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 valid_preds, avg_val_loss = self._val(eval_loader, model)
 print(f"epoch {epoch+1}/{n_epochs}")
 print(f"avg_loss: {avg_loss:.4f}")
 print(f"avg_val_loss: {avg_val_loss:.4f}")
 if best_score > avg_val_loss:
torch.save(model.state_dict() ,
self.path / f"best{self.fold_num}.pt")
 print(f"Save model on epoch {epoch + 1}")
 best_score = avg_val_loss
model.load_state_dict(torch.load(self.path / f"best{self.fold_num}.pt"))
valid_preds, avg_val_loss = self._val(eval_loader, model)
 print(f"Validation loss: {avg_val_loss}")
 return valid_preds
def _val(self, loader, model):
model.eval()
valid_preds = np.zeros(loader.dataset.cat_features.size(0))
avg_val_loss = 0.
 for i,(i_batch, c_batch, n_batch, y_batch)in enumerate(loader):
with torch.no_grad() :
i_batch = i_batch.to(self.device)
 y_pred = model(i_batch, c_batch, n_batch ).detach()
avg_val_loss += self.loss_fn(y_pred,
y_batch ).item() / len(loader)
 valid_preds[i * self.val_batch:(i + 1)* self.val_batch] = y_pred.cpu().numpy() [:, 0]
return valid_preds, avg_val_loss 
<choose_model_class>
test['Cabin']= test['Cabin'].str[:1]
data.Cabin.value_counts()
class NeuralNet(nn.Module):
def __init__(self, path, emb_dims, num_dims, img_linear, linear_size):
super(NeuralNet, self ).__init__()
self.densenet121 = models.densenet121()
self.densenet121.load_state_dict(torch.load(path))
self.densenet121.classifier = nn.Linear(1024, img_linear)
 dense = nn.Sequential(*list(self.densenet121.children())[:-1])
 for param in dense.parameters() :
param.requires_grad = False
self.embeddings = nn.ModuleList(
 [nn.Embedding(x, y)for x, y in emb_dims])
 n_emb_out = sum([y for x, y in emb_dims])
 self.fc1 = nn.Linear(img_linear + n_emb_out + num_dims, linear_size)
 self.bn1 = nn.BatchNorm1d(linear_size)
 self.fc2 = nn.Linear(linear_size, 1)
 self.drop = nn.Dropout(0.2)
 self.relu = nn.ReLU()
def forward(self, i, c, n):
img_feats = self.densenet121(i)
 emb = [
emb_layer(c[:, j])for j, emb_layer in enumerate(self.embeddings)
 ]
emb = torch.cat(emb, 1)
 data = torch.cat([img_feats, emb, n], 1)
 out = self.relu(self.fc1(data))
out = self.bn1(out)
 out = self.drop(out)
 out = self.fc2(out)
 return out<choose_model_class>
print('Data Set
',data.isnull().sum())
 print('-------xxxx-------')
 print('Test Set
',test.isnull().sum())
 data.Fare
emb_dims = [(2, 1),(n_breed1, 3),(n_breed2, 3),(3, 1),(n_color1, 1), 
(n_color2, 1),(n_color3, 1),(4, 1),(3, 1),(3, 1), 
(3, 1),(3, 1),(3, 1),(19, 1),(14, 1),(n_langs, 1)]
num_dims = X_test_ss.shape[1]
trainer = Trainer(
 NeuralNet,
resc=train_pet_ids,
n_splits=4,
train_batch=128,
val_batch=128,
kwargs={
"path": "densenet121.pth",
"emb_dims": emb_dims,
"num_dims": num_dims,
"img_linear": 48,
"linear_size": 120
} )<train_model>
train_test_data = [data, test]
for dataset in train_test_data:
dataset.loc[ dataset['Age'] <= 18, 'Age'] = 0,
dataset.loc[(dataset['Age'] > 18)&(dataset['Age'] <= 26), 'Age'] = 1,
dataset.loc[(dataset['Age'] > 26)&(dataset['Age'] <= 36), 'Age'] = 2,
dataset.loc[(dataset['Age'] > 36)&(dataset['Age'] <= 62), 'Age'] = 3,
dataset.loc[ dataset['Age'] > 62, 'Age'] = 4
for dataset in train_test_data:
dataset.loc[ dataset['Fare'] <= 20, 'Fare'] = 0,
dataset.loc[(dataset['Fare'] > 20)&(dataset['Fare'] <= 50), 'Fare'] = 1,
dataset.loc[(dataset['Fare'] > 50)&(dataset['Fare'] <= 100), 'Fare'] = 2,
dataset.loc[ dataset['Fare'] > 100, 'Fare'] = 3
trainer.fit(train_pet_ids, X_train_cat, X_train_ss, target, 5 )<create_dataframe>
k_fold = KFold(n_splits=10, shuffle=True, random_state=0 )
bin_path = trainer.path
test_preds = np.zeros(( X_test_cat.shape[0]))
c_tensor = torch.tensor(X_test_cat.values, dtype=torch.long ).to(trainer.device)
 n_tensor = torch.tensor(X_test_ss, dtype=torch.float32 ).to(trainer.device)
 test_dataset = PetDataset(test_pet_ids,
c_tensor,
n_tensor,
labels=None,
root_dir=".. /input/petfinder-adoption-prediction/test_images/",
transform=ds_trans)
 test_loader = data.DataLoader(test_dataset, batch_size=128, shuffle=False)
 for path in bin_path.iterdir() :
print(f"using {str(path)}")
 model = NeuralNet(**trainer.kwargs)
 model.to("cuda:0")
 model.load_state_dict(torch.load(path))
model.eval()
temp = np.zeros(( X_test_cat.shape[0]))
for i,(i_batch, c_batch, n_batch)in enumerate(test_loader):
i_batch = i_batch.to(trainer.device)
 with torch.no_grad() :
y_pred = model(i_batch, c_batch, n_batch ).detach()
temp[i * 128:(i + 1)* 128] = y_pred.cpu().numpy() [:, 0]
test_preds += temp / trainer.n_splits<categorify>
x_train,x_test,y_train,y_test = train_test_split(data[['Pclass','Sex','Age','Fare','Cabin','Embarked','Title']] ,data['Survived'] ,test_size = 0.35,random_state = 4 )
class ImageDataset(data.Dataset):
def __init__(self, pet_ids, root_dir, transform=None):
self.pet_ids = pet_ids
self.root_dir = root_dir
self.transform = transform
def __len__(self):
return len(self.pet_ids)
 def __getitem__(self, idx):
img_name = f"{self.pet_ids[idx]}-1.jpg"
fullname = self.root_dir / Path(img_name)
 try:
image = Image.open(fullname ).convert("RGB")
 except FileNotFoundError:
image = np.zeros(( 3, 224, 224), dtype=np.uint8 ).transpose(1, 2, 0)
 image = Image.fromarray(np.uint8(image))
if self.transform:
image = self.transform(image)
 return [image]<create_dataframe>
prediction = knn.predict(test[['Pclass','Sex','Age','Fare','Cabin','Embarked','Title']] )
train_dataset = ImageDataset(
 train_pet_ids,
".. /input/petfinder-adoption-prediction/train_images/",
transform=ds_trans)
 batch = 256
n_img_dim = 48
train_loader = data.DataLoader(train_dataset,
batch_size=batch,
shuffle=False)
 X_train_img = np.zeros(( len(train_pet_ids), n_img_dim))
test_dataset = ImageDataset(
 test_pet_ids,
".. /input/petfinder-adoption-prediction/test_images/",
transform=ds_trans)
 test_loader = data.DataLoader(test_dataset,
batch_size=batch,
shuffle=False)
 X_test_img = np.zeros(( len(test_pet_ids), n_img_dim))
bin_path = trainer.path
for path in bin_path.iterdir() :
model = NeuralNet(**trainer.kwargs)
 model.to("cuda:0")
 model.load_state_dict(torch.load(path))
model.eval()
temp = np.zeros(( len(train_pet_ids), n_img_dim))
for i,(i_batch,)in tqdm(enumerate(train_loader)) :
with torch.no_grad() :
i_batch = i_batch.to("cuda:0")
 y_pred = model.densenet121(i_batch ).detach()
temp[i * batch:(i + 1)* batch, :] = y_pred.cpu().numpy()
X_train_img += temp / trainer.n_splits
temp = np.zeros(( len(test_pet_ids), n_img_dim))
for i,(i_batch,)in tqdm(enumerate(test_loader)) :
with torch.no_grad() :
i_batch = i_batch.to("cuda:0")
 y_pred = model.densenet121(i_batch ).detach()
temp[i * batch:(i + 1)* batch, :] = y_pred.cpu().numpy()
X_test_img += temp / trainer.n_splits<create_dataframe>
sol = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": prediction
} )
train_img = pd.DataFrame(data=X_train_img, columns=[
f"img{i}" for i in range(X_train_img.shape[1])
 ])
 test_img = pd.DataFrame(data=X_test_img, columns=[
f"img{i}" for i in range(X_test_img.shape[1])
 ] )<concatenate>
sol.to_csv('sol6.csv', index=False )
num_columns = X_train_num.columns
X_train_num_df = pd.DataFrame(data=X_train_ss, columns=num_columns)
 X_test_num_df = pd.DataFrame(data=X_test_ss, columns=num_columns)
 X_test_num_df.index = X_test_cat.index
test_img.index = X_test_cat.index
X_train_all = pd.concat([X_train_num_df, X_train_cat, train_img], axis=1)
 X_test_all = pd.concat([X_test_num_df, X_test_cat, test_img], axis=1)
 print(X_train_all.shape, X_test_all.shape)
 X_train_all.head()<import_modules>
from sklearn.linear_model import LogisticRegression
class CategoryDataset(data.Dataset):
def __init__(self, category):
self.category = category
def __len__(self):
return len(self.category)
 def __getitem__(self, idx):
category = self.category[idx, :]
return [category]<prepare_x_and_y>
lr = LogisticRegression()
lr.fit(x_train,y_train)
 prediction3 = lr.predict(test[['Pclass','Sex','Age','Fare','Cabin','Embarked','Title']])
 metrics.accuracy_score(lr.predict(x_test),y_test )
c_train = torch.tensor(X_train_cat.values, dtype=torch.long ).to("cuda:0")
 c_test = torch.tensor(X_test_cat.values, dtype=torch.long ).to("cuda:0")
 train_dataset = CategoryDataset(c_train)
 test_dataset = CategoryDataset(c_test)
 train_loader = data.DataLoader(train_dataset, batch_size=128, shuffle=False)
 test_loader = data.DataLoader(test_dataset, batch_size=128, shuffle=False)
 X_train_cat_ = np.zeros(( len(train_pet_ids), 20))
for i,(c_batch,)in tqdm(enumerate(train_loader)) :
with torch.no_grad() :
y_pred = [model.embeddings[i](c_batch[:, i])for i in range(len(model.embeddings)) ]
y_pred =torch.cat(y_pred, 1 ).detach()
X_train_cat_[i * 128:(i + 1)* 128, :] = y_pred.cpu().numpy()
X_test_cat_ = np.zeros(( len(test_pet_ids), 20))
for i,(c_batch,)in tqdm(enumerate(test_loader)) :
with torch.no_grad() :
y_pred = [model.embeddings[i](c_batch[:, i])for i in range(len(model.embeddings)) ]
y_pred = torch.cat(y_pred, 1 ).detach()
X_test_cat_[i * 128:(i + 1)* 128, :] = y_pred.cpu().numpy()<create_dataframe>
sol4 = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": prediction3
} )
train_emb = pd.DataFrame(data=X_train_cat_, columns=[
f"emb{i}" for i in range(X_train_cat_.shape[1])
 ])
 test_emb = pd.DataFrame(data=X_test_cat_, columns=[
f"emb{i}" for i in range(X_test_cat_.shape[1])
 ] )<concatenate>
sol4.to_csv('sol.csv',index=False )
test_emb.index = X_test_all.index
X_train_all = pd.concat([X_train_all, train_emb], axis=1)
 X_test_all = pd.concat([X_test_all, test_emb], axis=1)
 print(X_train_all.shape, X_test_all.shape)
 X_train_all.head()<train_model>
sm = SVC()
sm.fit(data[['Pclass','Sex','Age','Fare','Cabin','Embarked','Title']],data['Survived'])
xgb_params = {
"eval_metric": "rmse",
"seed": 1337,
"eta": 0.01,
"subsample": 0.75,
"colsample_bytree": 0.85,
"tree_method": "gpu_hist",
"device": "gpu",
"silent": 1
}
xgb_X = X_train_all
xgb_y = target
xgb_X_test = X_test_all
model, oof_train_xgb, oof_test_xgb= run_xgb(
 xgb_params,
xgb_X,
xgb_y,
xgb_X_test,
resc=train_pet_ids,
n_splits=5,
num_rounds=10000 )<split>
pred_svm = sm.predict(test[['Pclass','Sex','Age','Fare','Cabin','Embarked','Title']])
 metrics.accuracy_score(sm.predict(x_test),y_test )
def run_lgb(params,
X,
y,
X_test,
resc,
cat_features,
n_splits=10,
early_stop=500):
fold = GroupKFold(n_splits=n_splits)
 oof_train = np.zeros(( X.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
for i,(trn_index, val_index)in enumerate(fold.split(X, y, resc)) :
X_tr = X.iloc[trn_index, :]
X_val = X.iloc[val_index, :]
y_tr = y[trn_index]
y_val = y[val_index]
model = lgb.LGBMRegressor(**params)
 model.fit(X_tr,
y_tr,
eval_set=(X_val, y_val),
verbose=500,
early_stopping_rounds=early_stop,
categorical_feature=cat_features)
 valid_pred = model.predict(X_val, num_iteration=model.best_iteration_)
 test_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 oof_train[val_index] = valid_pred
oof_test[:, i] = test_pred
return model, oof_train, oof_test<train_model>
subb = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": pred_svm
})
 subb.to_csv('submission1.csv', index=False )
<feature_engineering><EOS> 
rfc = RandomForestClassifier(n_estimators=14)
 rfc.fit(x_train,y_train)
 rfcpred = rfc.predict(test[['Pclass','Sex','Age','Fare','Cabin','Embarked','Title']])
(metrics.accuracy_score(rfc.predict(x_test),y_test))