Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<train_model>
plt.style.use("seaborn-whitegrid")
 plt.rc("figure", autolayout=True)
 plt.rc(
 "axes",
labelweight="bold",
labelsize="large",
titleweight="bold",
titlesize=14,
titlepad=10, 
)
 warnings.filterwarnings('ignore' )
@Cache('./cache')
 def build_trainer(trainer: MoaBase, df: pd.DataFrame, predictors: List[str], target_cols: List[str], num_seed_blends: int, n_splits: int, params: dict):
folds = splitter_for_moa(df, target_cols, n_splits=n_splits)
 model = trainer(
 target_cols = target_cols,
categorical_cols=[],
ignore_cols=[],
num_seed_blends=num_seed_blends,
metric=logloss_for_multilabel,
params=params,
exp=None 
)
 model.train(df[predictors], df[target_cols], folds)
 return model
stage1_1_tabnet_trainer = build_trainer(
 trainer=Tabnet,
df=train_features_df,
target_cols=stage_1_1_target_cols,
predictors=stage_1_train_features,
num_seed_blends=5,
n_splits=7,
params={}, 
)
 stage1_2_tabnet_trainer = build_trainer(
 trainer=Tabnet,
df=train_features_df,
target_cols=stage_1_2_target_cols,
predictors=stage_1_train_features,
num_seed_blends=5,
n_splits=7,
params={},
)<choose_model_class>
def load_data() :
data_dir = Path(".. /input/house-prices-advanced-regression-techniques/")
 df_train = pd.read_csv(data_dir / "train.csv", index_col="Id")
 df_test = pd.read_csv(data_dir / "test.csv", index_col="Id")
 df = pd.concat([df_train, df_test])
 df = clean(df)
 df = encode(df)
 df = impute(df)
 df_train = df.loc[df_train.index, :]
df_test = df.loc[df_test.index, :]
return df_train, df_test
stage1_2_xgb_trainer = build_trainer(
 trainer=XGBTrainer,
df=train_features_df,
target_cols=stage_1_2_target_cols,
predictors=stage_1_train_features,
num_seed_blends=2,
n_splits=5,
params={},
)<find_best_model_class>
data_dir = Path(".. /input/house-prices-advanced-regression-techniques/")
 df = pd.read_csv(data_dir / "train.csv", index_col="Id")
 df.Exterior2nd.unique()
folds = splitter_for_moa(train_features_df, stage_1_2_target_cols, n_splits=4)
 stage1_2_svm_trainer = SVMTrainer(
 target_cols = stage_1_2_target_cols,
categorical_cols=[],
ignore_cols=[],
num_seed_blends=1,
metric=logloss_for_multilabel,
params={
},
exp=None 
)
 stage1_2_svm_trainer.train_predict(
 X_train=train_features_df[stage_1_train_features],
y_train=train_features_df[stage_1_2_target_cols],
X_test=test_features_df[stage_1_train_features],
folds=folds
)<predict_on_test>
def clean(df):
df["Exterior2nd"] = df["Exterior2nd"].replace({"Brk Cmn": "BrkComm"})
 df["GarageYrBlt"] = df["GarageYrBlt"].where(df.GarageYrBlt <= 2010, df.YearBuilt)
 df.rename(columns={
"1stFlrSF": "FirstFlrSF",
"2ndFlrSF": "SecondFlrSF",
"3SsnPorch": "Threeseasonporch",
},inplace=True 
)
 return df
stage1_1_nn_pred = stage1_1_nn_trainer.predict(test_features_df)
 stage1_2_nn_pred = stage1_2_nn_trainer.predict(test_features_df)
 stage1_1_tabnet_pred = stage1_1_tabnet_trainer.predict(test_features_df)
 stage1_2_tabnet_pred = stage1_2_tabnet_trainer.predict(test_features_df)
 stage1_2_svm_pred = stage1_2_svm_trainer.result.preds
stage1_2_xgb_pred = stage1_2_xgb_trainer.predict(test_features_df )<concatenate>
features_nom = [
"MSSubClass",
"MSZoning",
"Street",
"Alley",
"LandContour",
"LotConfig",
"Neighborhood",
"Condition1",
"Condition2",
"BldgType",
"HouseStyle",
"RoofStyle",
"RoofMatl",
"Exterior1st",
"Exterior2nd",
"MasVnrType",
"Foundation",
"Heating",
"CentralAir",
"GarageType",
"MiscFeature",
"SaleType",
"SaleCondition"
]
five_levels = ["Po", "Fa", "TA", "Gd", "Ex"]
ten_levels = list(range(10))
ordered_levels = {
"OverallQual": ten_levels,
"OverallCond": ten_levels,
"ExterQual": five_levels,
"ExterCond": five_levels,
"BsmtQual": five_levels,
"BsmtCond": five_levels,
"HeatingQC": five_levels,
"KitchenQual": five_levels,
"FireplaceQu": five_levels,
"GarageQual": five_levels,
"GarageCond": five_levels,
"PoolQC": five_levels,
"LotShape": ["Reg", "IR1", "IR2", "IR3"],
"LandSlope": ["Sev", "Mod", "Gtl"],
"BsmtExposure": ["No", "Mn", "Av", "Gd"],
"BsmtFinType1": ["Unf", "LwQ", "Rec", "BLQ", "ALQ", "GLQ"],
"BsmtFinType2": ["Unf", "LwQ", "Rec", "BLQ", "ALQ", "GLQ"],
"Functional": ["Sal", "Sev", "Maj1", "Maj2", "Mod", "Min2", "Min1", "Typ"],
"GarageFinish": ["Unf", "RFn", "Fin"],
"PavedDrive": ["N", "P", "Y"],
"Utilities": ["NoSeWa", "NoSewr", "AllPub"],
"CentralAir": ["N", "Y"],
"Electrical": ["Mix", "FuseP", "FuseF", "FuseA", "SBrkr"],
"Fence": ["MnWw", "GdWo", "MnPrv", "GdPrv"]
}
ordered_levels = {key: ["None"] + value for key, value in
ordered_levels.items() }
def encode(df):
for name in features_nom:
df[name] = df[name].astype("category")
 if "None" not in df[name].cat.categories:
df[name].cat.add_categories("None", inplace=True)
 for name, levels in ordered_levels.items() :
df[name] = df[name].astype(CategoricalDtype(levels,
ordered=True))
return df
stack_train_df = pd.concat(
 [
pd.DataFrame(stage1_1_nn_trainer.result.oof_preds ).add_prefix("stage1_1_nn_"),
pd.DataFrame(stage1_2_nn_trainer.result.oof_preds ).add_prefix("stage1_2_nn_"),
train_features_df[stage_1_2_target_cols + ["drug_id", "sig_id"]],
],
axis=1, 
)
 stack_test_df = pd.concat(
 [
pd.DataFrame(stage1_1_nn_pred ).add_prefix("stage1_1_nn_"),
pd.DataFrame(stage1_2_nn_pred ).add_prefix("stage1_2_nn_"),
],
axis=1, 
)
 stack_train_predictors = [x for x in stack_train_df.columns if x not in stage_1_2_target_cols + ["drug_id", "sig_id"]]
stack_nn_trainer = train_nn_model(
 df=stack_train_df,
target_cols=stage_1_2_target_cols,
predictors=stack_train_predictors,
model_class=TabularMLP_2,
num_seed_blends=5,
n_splits=7 
)
 stacking_nn_pred = stack_nn_trainer.predict(stack_test_df )<concatenate>
def impute(df):
for name in df.select_dtypes("number"):
df[name] = df[name].fillna(0)
 for name in df.select_dtypes("category"):
df[name] = df[name].fillna("None")
 return df
stack_train_df = pd.concat(
 [
pd.DataFrame(stage1_1_tabnet_trainer.result.oof_preds ).add_prefix('stage1_1_tabnet_'),
pd.DataFrame(stage1_2_tabnet_trainer.result.oof_preds ).add_prefix('stage1_2_tabnet_'),
train_features_df[stage_1_2_target_cols + ["drug_id", "sig_id"]],
],
axis=1, 
)
 stack_test_df = pd.concat(
 [
pd.DataFrame(stage1_1_tabnet_pred ).add_prefix("stage1_1_tabnet_"),
pd.DataFrame(stage1_2_tabnet_pred ).add_prefix("stage1_2_tabnet_"),
],
axis=1, 
)
 stack_train_predictors = [x for x in stack_train_df.columns if x not in stage_1_2_target_cols + ["drug_id", "sig_id"]]
stack_tabnet_trainer = build_trainer(
 trainer=Tabnet,
df=stack_train_df,
target_cols=stage_1_2_target_cols,
predictors=stack_train_predictors,
num_seed_blends=5,
n_splits=7,
params={}, 
)
 stacking_tabnet_pred = stack_tabnet_trainer.predict(stack_test_df )<load_pretrained>
df_train, df_test = load_data()
stage1_2_trainers = {
'nn': stage1_2_nn_trainer,
'tabnet': stage1_2_tabnet_trainer,
'svm': stage1_2_svm_trainer,
'stack_nn': stack_nn_trainer,
'stack_tabnet': stack_tabnet_trainer,
'xgb': stage1_2_xgb_trainer,
}
for name, trainer in stage1_2_trainers.items() :
print(name, logloss_for_multilabel(train_features_df[stage_1_2_target_cols].values, trainer.result.oof_preds))<define_variables>
def score_dataset(X, y, model=XGBRegressor()):

 for colname in X.select_dtypes(["category"]):
X[colname] = X[colname].cat.codes
log_y = np.log(y)
 score = cross_val_score(
 model,
X,
log_y,
cv=5,
scoring="neg_mean_squared_error" 
)
 score = -1 * score.mean()
score = np.sqrt(score)
 return score
stage3_oofs = [
stack_nn_trainer.result.oof_preds,
stage1_2_svm_trainer.result.oof_preds,
stage1_2_tabnet_trainer.result.oof_preds,
stage1_2_xgb_trainer.result.oof_preds,
]
stage3_preds = [
stacking_nn_pred,
stage1_2_svm_pred,
stage1_2_tabnet_pred,
stage1_2_xgb_pred,
]<import_modules>
X = df_train.copy()
y = X.pop("SalePrice")
 baseline_score = score_dataset(X, y)
 print(f"Baseline score: {baseline_score:.5f} RMSLE" )
DEVICE = get_device()
class CNNStacking(nn.Module):
def __init__(self, n_features, n_labels):
super(CNNStacking, self ).__init__()
self.sq = nn.Sequential(
 nn.Conv2d(in_channels=1, out_channels=8, kernel_size=(3, 1), bias=False),
nn.ReLU() ,
nn.Conv2d(in_channels=8, out_channels=16, kernel_size=(2, 1), bias=False),
nn.ReLU() ,
nn.Flatten() ,
nn.Linear(in_features=16* n_labels, out_features=4 * n_labels),
nn.ReLU() ,
nn.Linear(in_features=4 * n_labels, out_features=n_labels), 
)
 def forward(self, x):
return self.sq(x)
 class CNNDataset(Dataset):
def __init__(self, X: np.ndarray, y: Optional[pd.DataFrame]):
self.X = X
if y is not None:
self.y = y.values
else:
self.y = y
def __len__(self):
return self.X.shape[0]
def __getitem__(self, idx):
if self.y is None:
return torch.tensor(self.X[idx], dtype=torch.float ).to(DEVICE)
 else:
return(
 torch.tensor(self.X[idx], dtype=torch.float ).to(DEVICE),
torch.tensor(self.y[idx], dtype=torch.float ).to(DEVICE), 
)
 class CNNTrainer(MoaBase):
def __init__(self, params: Optional[dict] = None, **kwargs):
if params is None:
self.params = {}
else:
self.params = params
super().__init__(**kwargs)
 def _get_default_params(self):
return {
'lr': 1e-4,
'batch_size': 256,
'epoch': 20,
}
def _train(self, X: pd.DataFrame, y: pd.DataFrame, predictors: List[str], train_idx: np.ndarray, valid_idx: np.ndarray, seed: int):
X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]
y_train, y_valid = y.iloc[train_idx], y.iloc[valid_idx]
target_cols = y_valid.columns.tolist()
_params = self._get_default_params()
_params.update(self.params)
 self.n_predictors = len(predictors)
 self.n_targets = len(target_cols)
 self.n_models = self.n_predictors // self.n_targets
num_epoch = _params['epoch']
batch_size = _params['batch_size']
net = CNNStacking(n_features=self.n_predictors, n_labels=self.n_targets)
 net.to(DEVICE)
 optimizer = RAdam(net.parameters() , lr=_params['lr'])
 valid_criterion = nn.BCEWithLogitsLoss()
criterion = SmoothBCEwLogits(smoothing=0.001)
 scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0.0001)
 X_train = X_train[predictors].values.reshape(-1, self.n_models, self.n_targets, 1 ).transpose(0, 3, 1, 2)
 X_valid = X_valid[predictors].values.reshape(-1, self.n_models, self.n_targets, 1 ).transpose(0, 3, 1, 2)
 train_dataset = CNNDataset(X_train, y_train)
 train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last=True)
 valid_dataset = CNNDataset(X_valid, y_valid)
 valid_dataloader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)
 bar = trange(num_epoch, desc=f"seed: {seed} train : {X_train.shape[0]} valid:{X_valid.shape[0]}====")
 train_loss = []
valid_loss = []
best_loss = np.inf
best_preds = None
best_loss_epoch = 1
for epoch in bar:
running_loss = []
valid_loss = []
net.train()
for x, y in train_dataloader:
x = x.to(DEVICE)
 y = y.to(DEVICE)
 optimizer.zero_grad()
out = net(x)
 loss = criterion(out, y)
 loss.backward()
running_loss.append(loss.item())
 optimizer.step()
scheduler.step()
preds_valid = []
_valid_loss = []
net.eval()
with torch.no_grad() :
for x, y in valid_dataloader:
x = x.to(DEVICE)
 y = y.to(DEVICE)
 out = net(x)
 loss = valid_criterion(out, y)
 preds_valid.append(out.sigmoid().detach().cpu().numpy())
 _valid_loss.append(loss.item())
 bar.set_postfix(
 running_loss=f"{np.mean(running_loss):.5f}",
valid_loss=f"{np.mean(_valid_loss):.5f}",
best_loss=f"{best_loss:.5f}",
best_loss_epoch=f"{best_loss_epoch}", 
)
 train_loss.append(np.mean(running_loss))
valid_loss.append(np.mean(_valid_loss))
if best_loss > np.mean(_valid_loss):
best_loss = np.mean(_valid_loss)
 best_loss_epoch = epoch + 1
best_preds = np.concatenate(preds_valid)
 best_state = copy.deepcopy(net.state_dict())
 logger.info(f"best loss : {best_loss}")
 model = CNNStacking(n_features=self.n_predictors, n_labels=self.n_targets)
 model.load_state_dict(best_state)
 model.to(DEVICE)
 return best_preds, model
def _predict(self, model: Any, X_valid: pd.DataFrame, predictors: List[str]):
_params = self._get_default_params()
_params.update(self.params)
 X_valid = X_valid[predictors].values.reshape(-1, self.n_models, self.n_targets, 1 ).transpose(0, 3, 1, 2)
 batch_size = _params['batch_size']
valid_dataset = CNNDataset(X_valid, None)
 valid_dataloader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)
 tmp_pred = []
model.eval()
with torch.no_grad() :
for x in valid_dataloader:
x = x.to(DEVICE)
 out = model(x)
 tmp_pred.append(out.sigmoid().detach().cpu().numpy())
 return np.concatenate(tmp_pred )<predict_on_test>
X = df_train.copy()
y = X.pop("SalePrice")
 mi_scores = make_mi_scores(X, y)
 mi_scores.head()
cnn_preds = cnn_trainer.predict(make_cnn_inputs(stage3_preds, stage_1_2_target_cols))<compute_train_metric>
def drop_uninformative(df, mi_scores):
return df.loc[:, mi_scores > 0.0]
def get_score(weights, train_idx, oofs, labels):
blend = np.zeros_like(oofs[0][train_idx, :])
 for oof, weight in zip(oofs[:-1], weights):
blend += weight * oof[train_idx, :]
blend +=(1 - np.sum(weights)) * oofs[-1][train_idx, :]
return logloss_for_multilabel(labels[train_idx, :], blend)
 def get_best_weights(oofs, labels):
weight_list = []
weights = np.array([1/len(oofs)for x in range(len(oofs)- 1)])
 for n_splits in tqdm([5, 6]):
for i in range(2):
kf = KFold(n_splits=n_splits, random_state=i, shuffle=True)
 for fold,(train_idx, valid_idx)in enumerate(kf.split(X=oofs[0])) :
res = minimize(get_score, weights, args=(train_idx, oofs, labels), method="Nelder-Mead", tol=1e-6)
 logger.info(f"i: {i} fold: {fold} res.x: {res.x}")
 weight_list.append(res.x)
 mean_weight = np.mean(weight_list, axis=0)
 print(f"optimized weight: {mean_weight}")
 return mean_weight
best_weights = get_best_weights(stage3_oofs, train_features_df[stage_1_2_target_cols].values )<categorify>
X = df_train.copy()
y = X.pop("SalePrice")
 X = drop_uninformative(X, mi_scores)
 score_dataset(X, y )
best_weights = np.insert(best_weights, len(best_weights), 1 - np.sum(best_weights))
oof_preds = np.stack(stage3_oofs ).transpose(1, 2, 0 ).dot(best_weights)
 print(logloss_for_multilabel(train_features_df[stage_1_2_target_cols].values, oof_preds))
blend_preds = np.stack(stage3_preds ).transpose(1, 2, 0 ).dot(best_weights )<feature_engineering>
def label_encode(df):
X = df.copy()
for colname in X.select_dtypes(["category"]):
X[colname] = X[colname].cat.codes
return X
final_blend_oofs = oof_preds * 0.7 + cnn_trainer.result.oof_preds * 0.3
test_features_df[stage_1_2_target_cols] = blend_preds * 0.7 + cnn_preds * 0.3<compute_test_metric>
def mathematical_transforms(df):
X = pd.DataFrame()
X["LivLotRatio"] = df.GrLivArea / df.LotArea
X["Spaciousness"] =(df.FirstFlrSF + df.SecondFlrSF)/ df.TotRmsAbvGrd
return X
def interactions(df):
X = pd.get_dummies(df.BldgType, prefix="Bldg")
 X = X.mul(df.GrLivArea, axis=0)
 return X
def counts(df):
X = pd.DataFrame()
X["PorchTypes"] = df[[
"WoodDeckSF",
"OpenPorchSF",
"EnclosedPorch",
"Threeseasonporch",
"ScreenPorch"
]].gt(0.0 ).sum(axis=1)
 return X
def break_down(df):
X = pd.DataFrame()
X["MSClass"] = df.MSSubClass.str.split("_", n=1, expand=True)[0]
return X
def group_transforms(df):
X = pd.DataFrame()
X["MedNhbdArea"] = df.groupby("Neighborhood")["GrLivArea"].transform("median")
 X["MedCondQual"] = df.groupby("OverallQual")["OverallCond"].transform("median")
 X["MedBsmt"] = df.groupby("BsmtQual")["TotalBsmtSF"].transform("median")
 return X
print(logloss_for_multilabel(train_features_df[stage_1_2_target_cols].values, final_blend_oofs))<save_to_csv>
def apply_pca(X, standardize=True):
if standardize:
X =(X - X.mean(axis=0)) / X.std(axis=0)
 pca = PCA()
X_pca = pca.fit_transform(X)
 component_names = [f"PC{i+1}" for i in range(X_pca.shape[1])]
X_pca = pd.DataFrame(X_pca, column=component_names)
 loadings = pd.DataFrame(
 pca.components_.T,
columns=component_names,
index=X.columns, 
)
 return pca, X_pca, loadings
def plot_variance(pca, width=8, dpi=100):
fig, axs = plt.subplots(1, 2)
 n = pca.n_components_
grid = np.arange(1, n + 1)
 evr = pca.explained_variance_ratio_
axs[0].bar(grid, evr)
 axs[0].set(
 xlabel="Component", title="% Explained Variance", ylim=(0.0, 1.0)
)
 cv = np.cumsum(evr)
 axs[1].plot(np.r_[0, grid], np.r_[0, cv], "o-")
 axs[1].set(
 xlabel="Component", title="% Cumulative Variance", ylim=(0.0, 1.0)
)
 fig.set(figwidth=8, dpi=100)
 return axs
test_features_df.loc[test_features_df['cp_type'] != 'trt_cp', stage_1_2_target_cols] = 0
sub = sample_submission_df.drop(columns=stage_1_2_target_cols ).merge(test_features_df[['sig_id']+stage_1_2_target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False)
 sub.head()<set_options>
def pca_inspired(df):
X = pd.DataFrame()
X["Feature1"] = df.GrLivArea + df.TotalBsmtSF
X["Feature2"] = df.YearRemodAdd * df.TotalBsmtSF
return X
def pca_components(df, features):
X = df.loc[:, features]
_, X_pca, _ = apply_pca(X)
 return X_pca
pca_features = [
"GarageArea",
"YearRemodAdd",
"TotalBsmtSF",
"GrLivArea",
]
warnings.simplefilter('ignore' )<define_variables>
def indicate_outliers(df):
X_new = pd.DataFrame()
X_new["Outlier"] =(df.Neighborhood == "Edwards")&(df.SaleCondition == "Partial")
 return X_new
root = './'
id_name = 'sig_id'
variance_threshould = 0.7
ncompo_genes = 80
ncompo_cells = 10
seed=817119<set_options>
class CrossFoldEncoder:
def __init__(self, encoder, **kwargs):
self.encoder_ = encoder
self.kwargs_ = kwargs
self.cv_ = KFold(n_splits=5)
 def fit_transform(self, X, y, cols):
self.fitted_encoders_ = []
self.cols_ = cols
X_encoded = []
for idx_encode, idx_train in self.cv_.split(X):
fitted_encoder = self.encoder_(cols=cols, **self.kwargs_)
 fitted_encoder.fit(
 X.iloc[idx_encode, :], y.iloc[idx_encode], 
)
 X_encoded.append(fitted_encoder.transform(X.iloc[idx_train, :])[cols])
 self.fitted_encoders_.append(fitted_encoder)
 X_encoded = pd.concat(X_encoded)
 X_encoded.columns = [name + "_encoded" for name in X_encoded.columns]
return X_encoded
def transform(self, X):
X_encoded_list = []
for fitted_encoder in self.fitted_encoders_:
X_encoded = fitted_encoder.transform(X)
 X_encoded_list.append(X_encoded[self.cols_])
 X_encoded = reduce(
 lambda x, y: x.add(y, fill_value=0), X_encoded_list 
)/ len(X_encoded_list)
 X_encoded.columns = [name + "_encoded" for name in X_encoded.columns]
return X_encoded
def Seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
Seed_everything(seed=42 )<categorify>
def create_features(df, df_test=None):
X = df.copy()
y = X.pop("SalePrice")
 mi_scores = make_mi_scores(X, y)
 if df_test is not None:
X_test = df_test.copy()
X_test.pop("SalePrice")
 X = pd.concat([X, X_test])
 X = drop_uninformative(X, mi_scores)
 X = X.join(mathematical_transforms(X))
X = X.join(interactions(X))
X = X.join(counts(X))
X = X.join(pca_inspired(X))
X = label_encode(X)
 if df_test is not None:
X_test = X.loc[df_test.index, :]
X.drop(df_test.index, inplace=True)
 encoder = CrossFoldEncoder(MEstimateEncoder, m=1)
 X = X.join(encoder.fit_transform(X, y, cols=["MSSubClass"]))
if df_test is not None:
X_test = X_test.join(encoder.transform(X_test))
if df_test is not None:
return X, X_test
else:
return X
df_train, df_test = load_data()
X_train = create_features(df_train)
 y_train = df_train.loc[:, "SalePrice"]
score_dataset(X_train, y_train )
def Metric(labels,preds):
labels = np.array(labels)
 preds = np.array(preds)
 metric = 0
for i in range(labels.shape[1]):
metric +=(-np.mean(labels[:,i]*np.log(np.maximum(preds[:,i],1e-15)) +(1-labels[:,i])*np.log(np.maximum(1-preds[:,i],1e-15))))
return metric/labels.shape[1]
def Write_log(logFile,text,isPrint=True):
if isPrint:
print(text)
 logFile.write(text)
 logFile.write('
')
 return None
def Rank_gauss_encoding(df,cols):
for col in cols:
print('rank gauss encoding:',col)
 tmp = df[col].rank().values - 1.0
tmp =(tmp / tmp.max())* 0.998 + 0.001
efi = np.sqrt(2.0)*erfinv(tmp)
 efi = efi - efi.mean()
df[col] = np.float16(efi)
 return df
def rationalApproximation(t):
c = [2.515517, 0.802853, 0.010328]
d = 1.432788, 0.189269, 0.001308
return t -(( c[2]*t + c[1])*t + c[0])/(((d[2]*t + d[1])*t + d[0])*t + 1.0)
 def normalCDFInverse(p):
if(p <= 0.0 or p >= 1.0):
assert(False)
 if(p < 0.5):
return -rationalApproximation(np.sqrt(-2.0*np.log(p)))
 return rationalApproximation(np.sqrt(-2.0*np.log(1-p)))
 def vdErfInvSingle(x):
if x == 0.0:
return 0.0
elif x < 0.0:
return -normalCDFInverse(-x)*0.7
else:
return normalCDFInverse(x)*0.7
def rankGauss(df, features):
df_size = df.shape[0]
for f in features:
if len(set(df[f])) == 1:
df[f] = 0
elif len(set(df[f])) == 2:
vals = sorted(list(set(df[f])))
 df[f] = df[f].replace(vals,[0,1])
 else:
df["rank"] =(df[f].rank(method="min")-1.0)/df_size*0.998+0.001
df[f] = df["rank"].apply(lambda x:vdErfInvSingle(x))
df[f] = df[f] - df[f].mean()
print("feature:%s rankGauss over"%(f))
if "rank" in df.columns.values.tolist() :
df.drop(["rank"],axis=1,inplace=True)
 print("rankGauss over")
 return df<load_from_csv>
X_train = create_features(df_train)
 y_train = df_train.loc[:, "SalePrice"]
xgb_params = dict(
 booster="gbtree",
max_depth=3,
learning_rate=0.01,
n_estimators=2180,
min_child_weight=5,
colsample_bytree=0.7,
subsample=0.7,
reg_alpha=0.2,
reg_lambda=0.006,
num_parallel_tree=5, 
)
 xgb = XGBRegressor(tree_method="gpu_hist", gpu_id=0, **xgb_params)
 score_dataset(X_train, y_train, xgb )
files = ['.. /input/lish-moa/test_features.csv',
'.. /input/lish-moa/train_targets_scored.csv',
'.. /input/lish-moa/train_features.csv',
'.. /input/lish-moa/train_targets_nonscored.csv',
'.. /input/lish-moa/train_drug.csv',
'.. /input/lish-moa/sample_submission.csv']
test = pd.read_csv(files[0])
 train_target = pd.read_csv(files[1])
 train = pd.read_csv(files[2])
 train_nonscored = pd.read_csv(files[3])
 train_drug = pd.read_csv(files[4])
 sub = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
<define_variables>
def objective(trial):
xgb_params = dict(
 max_depth=trial.suggest_int("max_depth", 2, 10),
learning_rate=trial.suggest_float("learning_rate", 1e-4, 1e-1, log=True),
n_estimators=trial.suggest_int("n_estimators", 1000, 8000),
min_child_weight=trial.suggest_int("min_child_weight", 1, 10),
colsample_bytree=trial.suggest_float("colsample_bytree", 0.2, 1.0),
subsample=trial.suggest_float("subsample", 0.2, 1.0),
reg_alpha=trial.suggest_float("reg_alpha", 1e-4, 1e2, log=True),
reg_lambda=trial.suggest_float("reg_lambda", 1e-4, 1e2, log=True), 
)
 xgb = XGBRegressor(tree_method="gpu_hist", gpu_id=0, **xgb_params)
 return score_dataset(X_train, y_train, xgb)
 study = optuna.create_study(direction="minimize")
 study.optimize(objective, n_trials=20)
 xgb_params = study.best_params
genes = [col for col in train.columns if col.startswith("g-")]
cells = [col for col in train.columns if col.startswith("c-")]
features = genes + cells
targets = [col for col in train_target if col!='sig_id'] 
<filter>
X_train, X_test = create_features(df_train, df_test)
 y_train = df_train.loc[:, "SalePrice"]
xgb = XGBRegressor(tree_method="gpu_hist", gpu_id=0, **xgb_params)
 xgb.fit(X_train, np.log(y))
predictions = np.exp(xgb.predict(X_test))
output = pd.DataFrame({'Id': X_test.index, 'SalePrice': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )