Kaggle competition: House Prices - Advanced Regression Techniques
ori_train = train.copy()
ctl_train = train.loc[train['cp_type']=='ctl_vehicle'].append(test.loc[test['cp_type']=='ctl_vehicle'] ).reset_index(drop=True)
 ctl_train2 = train.loc[train['cp_type']=='ctl_vehicle'].reset_index(drop=True)
 ori_test = test.copy()
ctl_test = test.loc[test['cp_type']=='ctl_vehicle'].reset_index(drop=True )<categorify>
warnings.filterwarnings('ignore' )
def Feature(df):
for col in ['cp_time','cp_dose']:
tmp = pd.get_dummies(df[col],prefix=col)
 df = pd.concat([df,tmp],axis=1)
 df.drop([col],axis=1,inplace=True)
 for col in genes:
if df[col].std() < 1.0:
df.drop([col],axis=1,inplace=True)
 print(col)
 genes.remove(col)
 df[genes+cells] = df[genes+cells]/10.0
df['gene_gt_0'] =(df[genes]>0.).mean(axis=1)
 df['gene_lt_0'] =(df[genes]<0.).mean(axis=1)
 df['cell_gt_0'] =(df[cells]>0.).mean(axis=1)
 df['cell_lt_0'] =(df[cells]<0.).mean(axis=1)
 df['gene_gt_0.7'] =(df[genes]>0.7 ).mean(axis=1)
 df['gene_lt_0.7'] =(df[genes]<0.7 ).mean(axis=1)
 df['cell_gt_0.7'] =(df[cells]>0.7 ).mean(axis=1)
 df['cell_lt_0.7'] =(df[cells]<0.7 ).mean(axis=1)
 df['gene_gt_0.9'] =(df[genes]>0.9 ).mean(axis=1)
 df['gene_lt_0.9'] =(df[genes]<0.9 ).mean(axis=1)
 df['cell_gt_0.9'] =(df[cells]>0.9 ).mean(axis=1)
 df['cell_lt_0.9'] =(df[cells]<0.9 ).mean(axis=1)
 pca_genes = PCA(n_components = ncompo_genes,
random_state = seed ).fit_transform(df[genes])
 pca_cells = PCA(n_components = ncompo_cells,
random_state = seed ).fit_transform(df[cells])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 df = pd.concat([df, pca_genes, pca_cells], axis = 1)
 pca_cols = [col for col in df.columns if 'pca' in col]
for col in pca_cols:
df[col] = df[col] / df[col].abs().max()
return df<feature_engineering>
def evaluation_metric(predicted_value, actual_value)-> float:

 rmse = mean_squared_error(actual_value,predicted_value, squared=False)
 return rmse
def verify_remodelling(remodelled_year:int, built_year:int):

 if remodelled_year > built_year:
return True
elif remodelled_year == built_year:
return False
else:
return -1
def selling_period(month_sold: int)-> str:

 if month_sold >= 3 and month_sold <= 5:
return 'spring'
elif month_sold >= 6 and month_sold <= 8:
return 'summer'
elif month_sold >= 9 and month_sold <= 11:
return 'autumn'
else:
return 'winter'
RANDOM_STATE = 0
def Feature1(df):

 for col in tqdm(genes+cells):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution='normal')
 transformer.fit(df[:train.shape[0]][col].values.reshape(-1,1))
df[col] = transformer.transform(df[col].values.reshape(-1,1)).reshape(1,-1)[0]
pca_genes = PCA(n_components = ncompo_genes,
random_state = 42 ).fit_transform(df[genes])
 pca_cells = PCA(n_components = ncompo_cells,
random_state = 42 ).fit_transform(df[cells])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 df = pd.concat([df, pca_genes, pca_cells], axis = 1)
 nor_var_col = [col for col in df.columns if col in ['sig_id','cp_type','cp_time','cp_dose'] or '_gt_' in col or '_lt_' in col]
var_thresh = VarianceThreshold(0.8)
 var_cols = [col for col in df.columns if col not in ['sig_id','cp_type','cp_time','cp_dose'] and '_gt_' not in col and '_lt_' not in col]
var_data = var_thresh.fit_transform(df[var_cols])
 df = pd.concat([df[nor_var_col],pd.DataFrame(var_data)],axis=1)

for col in ['cp_time','cp_dose']:
tmp = pd.get_dummies(df[col],prefix=col)
 df = pd.concat([df,tmp],axis=1)
 df.drop([col],axis=1,inplace=True)
 return df<feature_engineering>
train_set = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test_set = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
def Feature2(df):

 for col in tqdm(genes+cells):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution='normal')
 transformer.fit(df[:train.shape[0]][col].values.reshape(-1,1))
df[col] = transformer.transform(df[col].values.reshape(-1,1)).reshape(1,-1)[0]
gene_pca = PCA(n_components = ncompo_genes,
random_state = 42 ).fit(df[genes])
 pca_genes = gene_pca.transform(df[genes])
 cell_pca = PCA(n_components = ncompo_cells,
random_state = 42 ).fit(df[cells])
 pca_cells = cell_pca.transform(df[cells])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 df = pd.concat([df, pca_genes, pca_cells], axis = 1)

for col in ['cp_time','cp_dose']:
tmp = pd.get_dummies(df[col],prefix=col)
 df = pd.concat([df,tmp],axis=1)
 df.drop([col],axis=1,inplace=True)
 return df,transformer,gene_pca,cell_pca<concatenate>
train_set['total_square_footage'] = train_set['TotalBsmtSF'] + train_set['GrLivArea']
test_set['total_square_footage'] = test_set['TotalBsmtSF'] + test_set['GrLivArea']
train_set['remodelling_done'] = train_set.apply(lambda x: verify_remodelling(x['YearRemodAdd'], x['YearBuilt']), axis=1)
 test_set['remodelling_done'] = test_set.apply(lambda x: verify_remodelling(x['YearRemodAdd'], x['YearBuilt']), axis=1)
 train_set['selling_season'] = train_set.apply(lambda x: selling_period(x['MoSold']), axis=1)
 test_set['selling_season'] = test_set.apply(lambda x: selling_period(x['MoSold']), axis=1)
 train_set['total_area_of_house'] = train_set['TotalBsmtSF'] + train_set['1stFlrSF'] + train_set['2ndFlrSF']
test_set['total_area_of_house'] = test_set['TotalBsmtSF'] + test_set['1stFlrSF'] + test_set['2ndFlrSF']
train_set['age_of_building'] = train_set['YearBuilt'].apply(lambda x: pd.datetime.now().year - x)
 test_set['age_of_building'] = test_set['YearBuilt'].apply(lambda x: pd.datetime.now().year - x)
tt = train.append(test ).reset_index(drop=True)
 tt,transformer,gene_pca,cell_pca = Feature2(tt)
 train = tt[:train.shape[0]]
test = tt[train.shape[0]:].reset_index(drop=True )<drop_column>
train_set[ordinal_cat_features].isnull().sum()
if 1:
train_target = train_target.loc[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 ori_train = ori_train.loc[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 train = train.loc[train['cp_type']!='ctl_vehicle'].reset_index(drop=True )<normalization>
for col in ordinal_cat_features:
train_set[col] = train_set[col].map(ordinal_ranking)
 test_set[col] = test_set[col].map(ordinal_ranking)
 train_set[col] = train_set[col].fillna(0)
 test_set[col] = test_set[col].fillna(0 )
class resnetModel(nn.Module):
def __init__(self, num_features, num_targets, hidden_size,ispretrain=False):
super(resnetModel, self ).__init__()
self.ispretrain=ispretrain
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(num_features+hidden_size)
 self.dropout2 = nn.Dropout(0.2619422201258426)
 self.dense2 = nn.utils.weight_norm(nn.Linear(num_features+hidden_size, hidden_size))
self.batch_norm20 = nn.BatchNorm1d(hidden_size)
 self.dropout20 = nn.Dropout(0.2619422201258426)
 self.dense20 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(2*hidden_size)
 self.dropout3 = nn.Dropout(0.2619422201258426)
 self.dense3 = nn.utils.weight_norm(nn.Linear(2*hidden_size, hidden_size))
self.batch_norm30 = nn.BatchNorm1d(hidden_size)
 self.dropout30 = nn.Dropout(0.2619422201258426)
 self.dense30 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm4 = nn.BatchNorm1d(2*hidden_size)
 self.dropout4 = nn.Dropout(0.2619422201258426)
 if self.ispretrain:
self.dense4 = nn.utils.weight_norm(nn.Linear(2*hidden_size, num_targets))
else:
self.dense5 = nn.utils.weight_norm(nn.Linear(2*hidden_size, num_targets))
def forward(self, x):
x1 = self.batch_norm1(x)
 x1 = F.leaky_relu(self.dense1(x1))
x = torch.cat([x,x1],1)
 x2 = self.batch_norm2(x)
 x2 = self.dropout2(x2)
 x2 = F.leaky_relu(self.dense2(x2))
x2 = self.batch_norm20(x2)
 x2 = self.dropout20(x2)
 x2 = F.leaky_relu(self.dense20(x2))
x = torch.cat([x1,x2],1)
 x3 = self.batch_norm3(x)
 x3 = self.dropout3(x3)
 x3 = F.leaky_relu(self.dense3(x3))
x3 = self.batch_norm30(x3)
 x3 = self.dropout30(x3)
 x3 = F.leaky_relu(self.dense30(x3))
x3 = torch.cat([x2,x3],1)
 x3 = self.batch_norm4(x3)
 x3 = self.dropout4(x3)
 if self.ispretrain:
x3 = self.dense4(x3)
 else:
x3 = self.dense5(x3)
 return x3<define_search_model>
train_set[continious_features].isnull().sum()
class resnetModel2(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(resnetModel, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(num_features+hidden_size)
 self.dropout2 = nn.Dropout(0.2619422201258426)
 self.dense2 = nn.utils.weight_norm(nn.Linear(num_features+hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(2*hidden_size)
 self.dropout3 = nn.Dropout(0.2619422201258426)
 self.dense3 = nn.utils.weight_norm(nn.Linear(2*hidden_size, hidden_size))
self.batch_norm4 = nn.BatchNorm1d(2*hidden_size)
 self.dropout4 = nn.Dropout(0.2619422201258426)
 self.dense4 = nn.utils.weight_norm(nn.Linear(2*hidden_size, num_targets))
def forward(self, x):
x1 = self.batch_norm1(x)
 x1 = F.leaky_relu(self.dense1(x1))
x = torch.cat([x,x1],1)
 x2 = self.batch_norm2(x)
 x2 = self.dropout2(x2)
 x2 = F.leaky_relu(self.dense2(x2))
x = torch.cat([x1,x2],1)
 x3 = self.batch_norm3(x)
 x3 = self.dropout3(x3)
 x3 = self.dense3(x3)
 x3 = torch.cat([x2,x3],1)
 x3 = self.batch_norm4(x3)
 x3 = self.dropout4(x3)
 x3 = self.dense4(x3)
 return x3<normalization>
test_set[continious_features].isnull().sum()
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss 
<statistical_test>
imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')
 for col in continious_features:
imp_mean.fit(train_set[col].values.reshape(-1,1))
train_set[col] = imp_mean.transform(train_set[col].values.reshape(-1,1))
test_set[col] = imp_mean.transform(test_set[col].values.reshape(-1,1))
train_set[col] = np.log(train_set[col] + 1)
 test_set[col] = np.log(test_set[col] + 1 )
def Ctl_augment(train,target,include_test=0):
if include_test==0:
ctl_aug=ctl_train2.copy()
if include_test==1:
ctl_aug=ctl_train.copy()
aug_trains = []
aug_targets = []
for t in [24,48,72]:
for d in ['D1','D2']:
for _ in range(3):
train1 = train.loc[(train['cp_time']==t)&(train['cp_dose']==d)]
target1 = target.loc[(train['cp_time']==t)&(train['cp_dose']==d)]
ctl1 = ctl_aug.loc[(ctl_aug['cp_time']==t)&(ctl_aug['cp_dose']==d)].sample(train1.shape[0],replace=True)
 ctl2 = ctl_aug.loc[(ctl_aug['cp_time']==t)&(ctl_aug['cp_dose']==d)].sample(train1.shape[0],replace=True)
 train1[genes+cells] = train1[genes+cells].values + ctl1[genes+cells].values - ctl2[genes+cells].values
aug_train = train1.merge(target1,how='left',on='sig_id')
 aug_trains.append(aug_train[['cp_time','cp_dose']+genes+cells])
 aug_targets.append(aug_train[targets])
 df = pd.concat(aug_trains ).reset_index(drop=True)
 target = pd.concat(aug_targets ).reset_index(drop=True)
 for col in tqdm(genes+cells):
df[col] = transformer.transform(df[col].values.reshape(-1,1)).reshape(1,-1)[0]
pca_genes = gene_pca.transform(df[genes])
 pca_cells = cell_pca.transform(df[cells])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 df = pd.concat([df, pca_genes, pca_cells], axis = 1)
 for col in ['cp_time','cp_dose']:
tmp = pd.get_dummies(df[col],prefix=col)
 df = pd.concat([df,tmp],axis=1)
 df.drop([col],axis=1,inplace=True)
 xs = df[train_cols].values
ys = target[targets]
return xs,ys<categorify>
train_set[norminal_features].isnull().sum()
class MoADataset:
def __init__(self, features, targets,noise=0.1,val=0):
self.features = features
self.targets = targets
self.noise = noise
self.val = val
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
sample = self.features[idx, :].copy()
if 0 and np.random.rand() <0.3 and not self.val:
sample = self.swap_sample(sample)
 dct = {
'x' : torch.tensor(sample, dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
def swap_sample(self,sample):
num_samples = self.features.shape[0]
num_features = self.features.shape[1]
if len(sample.shape)== 2:
batch_size = sample.shape[0]
random_row = np.random.randint(0, num_samples, size=batch_size)
 for i in range(batch_size):
random_col = np.random.rand(num_features)< self.noise
sample[i, random_col] = self.features[random_row[i], random_col]
else:
batch_size = 1
random_row = np.random.randint(0, num_samples, size=batch_size)
 random_col = np.random.rand(num_features)< self.noise
sample[ random_col] = self.features[random_row, random_col]
return sample
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct 
<load_from_csv>
test_set[norminal_features].isnull().sum()
cs_df = pd.read_csv('.. /input/moa-cv-file/cv_fold.csv')
 cs_df=cs_df[cs_df['sig_id'].isin(train.sig_id)].reset_index(drop=True)
 device =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS1 = 29
EPOCHS = 23
trn_loss_=[]
def train_and_predict(features, sub, aug, folds=5, seed=817119,lr=1/90.0/3.5*3,weight_decay=1e-5/3):
dnn_oof = train[['sig_id']]
oof = train[['sig_id']]
for t in targets:
dnn_oof[t] = 0.0
oof[t] = 0.0
preds = []
test_X = test[features].values
test_data_loader = DataLoader(dataset=TensorDataset(torch.Tensor(test_X)) ,batch_size=1024,shuffle=False)
 for fold,(trn_ind, val_ind)in enumerate(StratifiedKFold(n_splits = folds, shuffle=True, random_state=seed)\ 
.split(train, cs_df['most_cs_same_target'])) :
train_X = train.loc[trn_ind,features].values
train_Y = train_target.loc[trn_ind,targets].values
train_Y = train_target.loc[trn_ind,targets].values
if 0:
aug_X,aug_Y = Ctl_augment(ori_train.loc[trn_ind],train_target.loc[trn_ind],include_test=1)
 train_X = np.concatenate([train_X,aug_X],axis=0)
 train_Y = np.concatenate([train_Y,aug_Y],axis=0)
 valid_X = train.loc[val_ind,features].values
valid_Y = train_target.loc[val_ind,targets].values

 valid_dataset = MoADataset(valid_X, valid_Y,val=1)
 valid_data_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=1024, shuffle=False)
 model = resnetModel(len(features),len(targets),1500)
 model.to(device)
 aug_X,aug_Y = Ctl_augment(ori_train.loc[trn_ind],train_target.loc[trn_ind],include_test=1)
 train_X_ = np.concatenate([train_X,aug_X],axis=0)
 train_Y_ = np.concatenate([train_Y,aug_Y],axis=0)
 train_dataset = MoADataset(train_X_, train_Y_)
 train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=128, shuffle=True)
 optimizer = torch.optim.Adam(model.parameters() ,betas=(0.9, 0.99), lr=1e-3, weight_decay=weight_decay,eps=1e-5)
 scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=lr, epochs=EPOCHS1, steps_per_epoch=len(train_data_loader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 best_valid_metric = 1e9
not_improve_epochs = 0
for epoch in range(EPOCHS1):
train_loss = 0.0
train_num = 0
for data in(train_data_loader):
optimizer.zero_grad()
x,y = data['x'].to(device),data['y'].to(device)
 outputs = model(x)
 loss = loss_tr(outputs, y)
 loss.backward()
optimizer.step()
scheduler.step()
train_num += x.shape[0]
train_loss +=(loss.item() *x.shape[0])
 train_loss /= train_num
model.eval()
valid_loss = 0.0
valid_num = 0
for data in(valid_data_loader):
x,y = data['x'].to(device),data['y'].to(device)
 outputs = model(x)
 loss = loss_fn(outputs, y)
 valid_num += x.shape[0]
valid_loss +=(loss.item() *x.shape[0])
 valid_loss /= valid_num
t_preds = []
for data in(test_data_loader):
x = data[0].to(device)
 with torch.no_grad() :
outputs = model(x)
 t_preds.extend(list(outputs.sigmoid().cpu().detach().numpy()))
pred_mean = np.mean(t_preds)
 if valid_loss < best_valid_metric:
torch.save(model.state_dict() ,'./model_resnet_fold%s.ckpt'%fold)
 not_improve_epochs = 0
best_valid_metric = valid_loss
print('[epoch %s] lr: %.6f, train_loss: %.6f, valid_metric: %.6f, pred_mean:%.6f'%(epoch,optimizer.param_groups[0]['lr'],train_loss,valid_loss,pred_mean))
trn_loss_.append(train_loss)
 else:
not_improve_epochs += 1
print('[epoch %s] lr: %.6f, train_loss: %.6f, valid_metric: %.6f, pred_mean:%.6f, NIE +1 ---> %s'%(epoch,optimizer.param_groups[0]['lr'],train_loss,valid_loss,pred_mean,not_improve_epochs))
if not_improve_epochs >= 50:
break
model.train()
if epoch!=EPOCHS1-1:
aug_X,aug_Y = Ctl_augment(ori_train.loc[trn_ind],train_target.loc[trn_ind],include_test=1)
 train_X_ = np.concatenate([train_X,aug_X],axis=0)
 train_Y_ = np.concatenate([train_Y,aug_Y],axis=0)
 train_dataset = MoADataset(train_X_, train_Y_)
 train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=128, shuffle=True)
 model = resnetModel(len(features),len(targets),1500)
 model.to(device)
 state_dict = torch.load('./model_resnet_fold%s.ckpt'%fold, torch.device("cuda" if torch.cuda.is_available() else "cpu"))
model.load_state_dict(state_dict)
 model.eval()
valid_preds = []
for data in tqdm(valid_data_loader):
x,y = data['x'].to(device),data['y'].to(device)
 with torch.no_grad() :
outputs = model(x)
 valid_preds.extend(list(outputs.cpu().detach().numpy()))
dnn_oof.loc[val_ind,targets] = 1 /(1+np.exp(-np.array(valid_preds)))
 t_preds = []
for data in tqdm(test_data_loader):
x = data[0].to(device)
 with torch.no_grad() :
outputs = model(x)
 t_preds.extend(list(outputs.sigmoid().cpu().detach().numpy()))
print(np.mean(t_preds))
preds.append(t_preds)
 sub[targets] = np.array(preds ).mean(axis=0)
 return dnn_oof,oof,sub<sort_values>
for col in norminal_features:
train_set[col] = train_set[col].astype(str)
 test_set[col] = test_set[col].astype(str)
 train_set[col] = train_set[col].fillna('Unknown')
 test_set[col] = test_set[col].fillna('Unknown' )
train_cols = [col for col in train.columns if col not in ['sig_id','cp_type','cs','most_cs_ind','most_cs_same_target']]
len(train_cols )<predict_on_test>
ohe = OneHotEncoder(handle_unknown = "ignore", sparse=False)
 ohe_train_set = pd.DataFrame(ohe.fit_transform(train_set[norminal_features]))
ohe_test_set= pd.DataFrame(ohe.transform(test_set[norminal_features]))
ohe_train_set.index = train_set.index
ohe_test_set.index = test_set.index
Seed_everything(817119)
 dnn_oof,oof,sub = train_and_predict(train_cols,sub.copy() ,aug=True,seed=817119,lr=1/90.0/3.5*4,weight_decay=1e-5/4)
<create_dataframe>
train_set = train_set.drop(norminal_features, axis = 1)
 test_set = test_set.drop(norminal_features, axis = 1 )
valid_metric = Metric(train_target[targets].values,dnn_oof[targets].values)
<feature_engineering>
train_base = train_set.copy()
test_base = test_set.copy()
train_set = pd.concat([train_set, ohe_train_set], axis=1)
 test_set = pd.concat([test_set, ohe_test_set], axis =1 )
sub.loc[test['cp_type']=='ctl_vehicle',targets] = 0.0<save_to_csv>
test_set[discrete_features].isnull().sum()
dnn_oof.to_csv('./oof.csv',index=False)
 sub.to_csv('./submission.csv',index=False )<define_variables>
train_set[discrete_features].isnull().sum()
test_set[discrete_features].isnull().sum()
for col in discrete_features:
test_set[col] = test_set[col].fillna(0 )
DO_SUBMISSION = True
DO_TRAIN_FOR_ENSEMBLE = False
DO_VIRTUAL_SUBMISSION = False
assert(sum([DO_SUBMISSION, DO_TRAIN_FOR_ENSEMBLE, DO_VIRTUAL_SUBMISSION])== 1), "select `ONE` mode"<import_modules>
correlation_features = corr_cols.append(corr_cols_small ).drop('SalePrice' )
import os
import gc
import sys
import time
import shutil
from pathlib import Path
from copy import deepcopy
from contextlib import contextmanager
import random
import typing as tp
from collections import OrderedDict
import yaml
import numpy as np
import pandas as pd
from sklearn.metrics import log_loss
from sklearn.model_selection import train_test_split<define_variables>
target = sale_price
test_set_Id = test_set['Id']
train = train_set.drop(columns=['SalePrice', 'Id'])
 test_set = test_set.drop(columns = 'Id' )
def generate_virtual_private(test_feat, smpl_sub, rate_to_pub=4):
assert(test_feat.sig_id == smpl_sub.sig_id ).all()
test_feat_list = [test_feat]
smpl_sub_list = [smpl_sub]
for i in range(1, rate_to_pub):
tmp_test_feat = test_feat.copy()
tmp_smpl_sub = smpl_sub.copy()
tmp_test_feat.sig_id = tmp_test_feat.sig_id + f"_{i}"
tmp_smpl_sub.sig_id = tmp_smpl_sub.sig_id + f"_{i}"
assert(tmp_test_feat.sig_id == tmp_smpl_sub.sig_id ).all()
tmp_test_feat.iloc[:, 4:] += i * 10
assert(tmp_test_feat.iloc[:, 4:] != test_feat.iloc[:, 4:] ).all().all()
test_feat_list.append(tmp_test_feat)
 smpl_sub_list.append(tmp_smpl_sub)
 test_feat_concat = pd.concat(test_feat_list, axis=0, ignore_index=True)
 smpl_sub_concat = pd.concat(smpl_sub_list, axis=0, ignore_index=True)
 return test_feat_concat, smpl_sub_concat<install_modules>
model_rmse_val = []
model_rmse_train = []
model_type = []
model_train_time = []
model_pred_time = []
%%time
!pip install.. /input/iterative-stratification/iterative-stratification-master/<install_modules>
train_x, val_x, train_y, val_y = train_test_split(train, target, train_size = 0.8, random_state = RANDOM_STATE )
%%time
!pip install --no-index --find-links.. /input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet<load_pretrained>
std_scaler = StandardScaler()
std_scaler.fit(train_x)
 train_x = std_scaler.transform(train_x)
 val_x = std_scaler.transform(val_x)
 test_set = std_scaler.transform(test_set)
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/nn-inference-0.01833.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-nn-use-train-public-inference.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<load_pretrained>
base_features = ['LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea','GarageArea',
'WoodDeckSF', 'OpenPorchSF','EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea','MiscVal','LotFrontage', 'age_of_building','total_square_footage',
'total_area_of_house','YearBuilt','YearRemodAdd','BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr','TotRmsAbvGrd','Fireplaces','GarageCars',
'MoSold', 'YrSold','LotShape', 'Utilities','LandSlope','ExterQual','ExterCond','BsmtQual','BsmtCond', 'BsmtExposure','BsmtFinType1','BsmtFinType2','HeatingQC'
,'KitchenQual', 'Functional','FireplaceQu','GarageFinish','GarageQual','GarageCond','PoolQC','Fence','OverallQual','OverallCond' ]
%%time
if DO_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/tabnet-inference-0.01840.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-nn-tabnet-fix5/virtual-tabnet-inference-add-param-n-shared.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<install_modules>
train_x_base, val_x_base, train_y_base, val_y_base = train_test_split(train[base_features], target, train_size = 0.75, random_state = RANDOM_STATE )
%%time
if DO_SUBMISSION:
!python.. /input/moa-takapy-script/tf-rn-transfer-1layerother-selcol100.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-takapy-script/virtual-tf-rn-transfer-1layerother-selcol100.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<install_modules>
std_scaler_base = StandardScaler()
std_scaler_base.fit(train_x_base)
 train_x_base = std_scaler_base.transform(train_x_base)
 val_x_base = std_scaler_base.transform(val_x_base )
%%time
if DO_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_VIRTUAL_SUBMISSION:
!python.. /input/moa-tawara-scripts-for-final-submission/virtual-moa-for-final-thrnn-seed-cv-0.01836.py
elif DO_TRAIN_FOR_ENSEMBLE:
pass
else:
raise ValueError<define_variables>
logreg = LinearRegression()
start_time = time.time()
logreg.fit(train_x_base, train_y_base)
 time_lapse = time.time() - start_time
model_type.append('LogReg Baseline')
 model_train_time.append(time_lapse)
 time_lapse
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']<load_from_csv>
logreg_pred_train = logreg.predict(train_x_base)
 logreg_train_rmse = evaluation_metric(logreg_pred_train, train_y_base)
 model_rmse_train.append(logreg_train_rmse)
 logreg_train_rmse
def del_control(df: pd.DataFrame())-> pd.DataFrame() :
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 df['cp_type'] = train['cp_type']
df = df.loc[df['cp_type']=='trt_cp'].reset_index(drop=True)
 df = df.drop('cp_type', axis=1)
 return df
def drop_sig_id(df: pd.DataFrame())-> pd.DataFrame() :
return df.drop('sig_id', axis=1)
 def fix_oof(df: pd.DataFrame())-> pd.DataFrame() :
if df.shape[0] != 21948:
df = del_control(df)
 if 'sig_id' in df.columns.tolist() :
df = drop_sig_id(df)
 return df<define_variables>
start_time = time.time()
logreg_pred_val = logreg.predict(val_x_base)
 time_lapse = time.time() - start_time
model_pred_time.append(time_lapse )
oof_list = []<load_from_csv>
%time logreg_val_rmse = evaluation_metric(logreg_pred_val, val_y_base )
oof_list.append(fix_oof(
 pd.read_pickle('.. /input/nn-use-train-public/oof.pkl')))
 oof_list.append(fix_oof(
 pd.DataFrame(
 np.load('.. /input/tabnet-train-public-add-n-shared-1/oof_tabnet.npy'),
columns=TARGET_COL)) 
)
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-takapy-tf-resnet-transfer/oof_prediction_takapy_resnet.csv")))
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-weight-thrnn-seed-cv/oof_prediction.csv")))
 oof_list.append(fix_oof(
 pd.read_csv(".. /input/moa-weight-thrnn-drug-seed-cv/oof_prediction.csv")))
<load_from_csv>
model_rmse_val.append(logreg_val_rmse)
 logreg_val_rmse
y_true = fix_oof(pd.read_csv('.. /input/lish-moa/train_targets_scored.csv'))<compute_test_metric>
result = pd.DataFrame(
 {
'feature': model_type,
'train time': model_train_time,
'validation prediction time':model_pred_time,
'rmse train':model_rmse_train,
'rmse validation':model_rmse_val
} 
)
 result.sort_values(by='rmse validation' )
print("sinchir0 - NN:", oof_list[0].shape)
 print("local logloss:", log_loss(y_true.values.ravel() , oof_list[0].values.ravel()))
display(oof_list[0].head())
 print("sinchir0 - TabNet:", oof_list[1].shape)
 print("local logloss:", log_loss(y_true.values.ravel() , oof_list[1].values.ravel()))
display(oof_list[1].head())
 print("takapy - ResNet:", oof_list[2].shape)
 print("local logloss:", log_loss(y_true.values.ravel() , oof_list[2].values.ravel()))
display(oof_list[2].head())
 print("tawara - ThrNN seed-cv:", oof_list[3].shape)
 print("local logloss:", log_loss(y_true.values.ravel() , oof_list[3].values.ravel()))
display(oof_list[3].head())
 print("tawara - ThrNN drug-seed-cv:", oof_list[4].shape)
 print("local logloss:", log_loss(y_true.values.ravel() , oof_list[4].values.ravel()))
display(oof_list[4].head() )<prepare_output>
train_x_corr, val_x_corr, train_y_corr, val_y_corr = train_test_split(train[correlation_features], target, train_size = 0.8, random_state = RANDOM_STATE)
 logreg_corr = LinearRegression()
start_time = time.time()
logreg_corr.fit(train_x_corr, train_y_corr)
 time_lapse = time.time() - start_time
model_type.append('LogReg - Correlation Features')
 model_train_time.append(time_lapse)
 time_lapse
Blend_oof = oof_list[0].copy()
Blend_oof[:] = 0.
 for oof_df in oof_list:
Blend_oof += oof_df
Blend_oof = Blend_oof / len(oof_list )<load_from_csv>
logreg_corr_pred_train = logreg_corr.predict(train_x_corr)
 logreg_corr_train_rmse = evaluation_metric(logreg_corr_pred_train, train_y_corr)
 model_rmse_train.append(logreg_corr_train_rmse)
 logreg_corr_train_rmse
y_true = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 y_true = fix_oof(y_true)
 y_pred = Blend_oof.copy()
print("CV log_loss: ", log_loss(y_true.values.ravel() , y_pred.values.ravel()))<load_from_csv>
start_time = time.time()
logreg_corr_pred_val = logreg_corr.predict(val_x_corr)
 time_lapse = time.time() - start_time
model_pred_time.append(time_lapse )
if DO_SUBMISSION or DO_VIRTUAL_SUBMISSION:
sub_list = [
pd.read_csv('./submission-sinchir0-nn.csv'),
pd.read_csv('./submission_sinchir0_tabnet.csv'),
pd.read_csv("./submission_takapy_tf-resnet.csv"),
pd.read_csv('./submission_tawara_thrnn_seed_cv.csv'),
pd.read_csv('./submission_tawara_thrnn_drug_seed_cv.csv'),
]
else:
sub_list = [
pd.read_csv('.. /input/nn-use-train-public/submission.csv'),
pd.read_csv('.. /input/tabnet-train-public-add-n-shared-1/submission.csv'),
pd.read_csv(".. /input/moa-takapy-tf-resnet-transfer/submission.csv"),
pd.read_csv('.. /input/moa-weight-thrnn-seed-cv/submission.csv'),
pd.read_csv('.. /input/moa-weight-thrnn-drug-seed-cv/submission.csv'),
]<sort_values>
logreg_corr_val_rmse = evaluation_metric(logreg_corr_pred_val, val_y_corr)
 model_rmse_val.append(logreg_corr_val_rmse)
 logreg_corr_val_rmse
def order_sub(sub):
return sub.sort_values('sig_id' ).reset_index(drop=True )<sort_values>
result = pd.DataFrame(
 {
'feature': model_type,
'train time': model_train_time,
'validation prediction time':model_pred_time,
'rmse train':model_rmse_train,
'rmse validation':model_rmse_val
} 
)
 result.sort_values(by='rmse validation' )
sub_list = [order_sub(sub_df)for sub_df in sub_list]<categorify>
def make_mi_scores(train_data, target_data):
mi_scores = mutual_info_regression(train_data, target_data)
 mi_scores = pd.Series(mi_scores, name="MI Scores", index=train.columns)
 mi_scores = mi_scores.sort_values(ascending=False)
 return mi_scores
mi_scores = make_mi_scores(train_x, train_y)
 mi_scores
class MoAStackingDataset(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index] ).float() ,
torch.from_numpy(self.label[index] ).float()
]
class MoAStackingDataset2d(data.Dataset):
def __init__(self, feat: np.ndarray, label: np.ndarray = None):

 self.feat = feat
if label is None:
self.label = np.full(( len(feat), 1), -1)
 else:
self.label = label
self.reset_model_order()
def reset_model_order(self):
self.model_order = np.arange(self.feat.shape[-1])
 def shuffle_model_order(self, seed):
np.random.seed(seed)
 self.model_order = np.random.permutation(self.model_order)
 def __len__(self):

 return len(self.feat)
 def __getitem__(self, index: int):

 return [
torch.from_numpy(self.feat[index][:, self.model_order] ).float() ,
torch.from_numpy(self.label[index] ).float()
]<define_search_model>
mi_features = mi_scores[mi_scores > 0.1].index.values
def get_activation(activ_name: str="relu"):

 act_dict = {
"relu": nn.ReLU() ,
"tanh": nn.Tanh() ,
"sigmoid": nn.Sigmoid() ,
"identity": nn.Identity() }
if activ_name in act_dict:
return act_dict[activ_name]
elif re.match(r"^htanh\_\d{4}$", activ_name):
bound = int(activ_name[-4:])/ 1000
return nn.Hardtanh(-bound, bound)
 else:
raise NotImplementedError
class LBAD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LBAD, self ).__init__()
layers = [nn.Linear(in_features, out_features)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
layers.append(get_activation(activ))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class BDLA(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(BDLA, self ).__init__()
layers = []
if use_bn:
layers.append(nn.BatchNorm1d(in_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
layers.append(nn.Linear(in_features, out_features))
if use_wn:
layers[-1] = nn.utils.weight_norm(layers[-1])
 layers.append(get_activation(activ))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class LABD(nn.Module):

 def __init__(
 self, in_features: int, out_features: int, drop_rate: float=0.0,
use_bn: bool=False, use_wn: bool=False, activ: str="relu" 
):

 super(LABD, self ).__init__()
layers = [nn.Linear(in_features, out_features), get_activation(activ)]
if use_wn:
layers[0] = nn.utils.weight_norm(layers[0])
 if use_bn:
layers.append(nn.BatchNorm1d(out_features))
if drop_rate > 0:
layers.append(nn.Dropout(drop_rate))
self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x )<choose_model_class>
train_x_mi, val_x_mi, train_y_mi, val_y_mi = train_test_split(train[mi_features], target, train_size = 0.8, random_state = RANDOM_STATE)
 logreg_mi = LinearRegression()
start_time = time.time()
logreg_mi.fit(train_x_mi, train_y_mi)
 time_lapse = time.time() - start_time
model_type.append('LogReg - Mutual Information Features')
 model_train_time.append(time_lapse)
 time_lapse
class MLP(nn.Module):

 def __init__(
 self, n_features_list: tp.List[int], use_tail_as_out: bool=False,
drop_rate: float=0.0, use_bn: bool=False, use_wn: bool=False,
activ:str="relu", block_name: str="LBAD" 
):

 super(MLP, self ).__init__()
n_layers = len(n_features_list)- 1
block_class = {
"LBAD": LBAD, "BDLA": BDLA, "LABD": LABD}[block_name]
layers = []
for i in range(n_layers):
in_feats, out_feats = n_features_list[i: i + 2]
if i == n_layers - 1 and use_tail_as_out:
if block_name in ["BDLA"]:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, "identity")
 else:
layer = nn.Linear(in_feats, out_feats)
 if use_wn:
layer = nn.utils.weight_norm(layer)
 else:
layer = block_class(in_feats, out_feats, drop_rate, use_bn, use_wn, activ)
 layers.append(layer)
 self.layers = nn.Sequential(*layers)
 def forward(self, x: torch.Tensor)-> torch.Tensor:

 return self.layers(x)
 class CNNForStacking(nn.Module):

 def __init__(
 self, n_models: int,
n_channels_list: tp.List[int],
kwargs_head: tp.Dict, 
):

 super(CNNForStacking, self ).__init__()
self.n_conv_layers = len(n_channels_list)- 1
for i in range(self.n_conv_layers):
in_ch = n_channels_list[i]
out_ch = n_channels_list[i + 1]
layer = nn.Sequential(
 nn.Conv1d(in_ch, out_ch, 3, 1, 0),
nn.BatchNorm1d(out_ch),
nn.ReLU(inplace=True))
setattr(self, "conv{}".format(i + 1), layer)
 kwargs_head["n_features_list"][0] =(n_models - 2 * self.n_conv_layers)* n_channels_list[-1]
self.head = MLP(**kwargs_head)
 def forward(self, x: torch.FloatTensor)-> torch.Tensor:

 bs = x.shape[0]
h = x
for i in range(self.n_conv_layers):
h = getattr(self, "conv{}".format(i + 1))(h)
 h = torch.reshape(h,(bs, -1))
h = self.head(h)
 return h<set_options>
logreg_mi_pred_train = logreg_mi.predict(train_x_mi)
 logreg_mi_train_rmse = evaluation_metric(logreg_mi_pred_train, train_y_mi)
 model_rmse_train.append(logreg_mi_train_rmse)
 logreg_mi_train_rmse
def set_random_seed(seed: int = 42, deterministic: bool = False):

 random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 if deterministic:
torch.backends.cudnn.deterministic = True<predict_on_test>
start_time = time.time()
logreg_mi_pred_val = logreg_mi.predict(val_x_mi)
 time_lapse = time.time() - start_time
model_pred_time.append(time_lapse )
def inference_function(settings, model, loader, device):
model.to(device)
 model.eval()
pred_list = []
with torch.no_grad() :
for x, t in loader:
y = model(x.to(device))
pred_list.append(y.sigmoid().detach().cpu().numpy())
 pred_arr = np.concatenate(pred_list)
 del pred_list
return pred_arr<concatenate>
logreg_mi_val_rmse = evaluation_metric(logreg_mi_pred_val, val_y_mi)
 model_rmse_val.append(logreg_mi_val_rmse)
 logreg_mi_val_rmse
X = np.concatenate([df.values for df in oof_list], axis=1)
 print(X.shape)
 y = y_true.values
print(y.shape)
 train_all_dataset = MoAStackingDataset(X, y)
 X_test = np.concatenate([
sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=1)
 print(X_test.shape)
 test_dataset = MoAStackingDataset(X_test, None)
 X_2d = np.stack([df.values for df in oof_list], axis=2)
 print(X_2d.shape)
 train_all_dataset_2d = MoAStackingDataset2d(X_2d, y)
 X_test_2d = np.stack(
 [sub_df.iloc[:, 1:].values for sub_df in sub_list], axis=2)
 print(X_test_2d.shape)
 test_dataset_2d = MoAStackingDataset2d(X_test_2d, None )<init_hyperparams>
result = pd.DataFrame(
 {
'feature': model_type,
'train time': model_train_time,
'validation prediction time':model_pred_time,
'rmse train':model_rmse_train,
'rmse validation':model_rmse_val
} 
)
 result.sort_values(by='rmse validation' )
settings_str =
settings = yaml.safe_load(settings_str)
 settings["model"]["params"]["n_features_list"][0] = len(oof_list)* 206<feature_engineering>
lreg = Lasso(random_state = RANDOM_STATE)
 start_time = time.time()
lreg.fit(train_x, train_y)
 time_lapse = time.time() - start_time
model_type.append('Lasso')
 model_train_time.append(time_lapse)
 time_lapse
device = torch.device(settings["globals"]["device"])
 cv_score_list_mlp = []
stgs_list_mlp = []
for tmp_seed in settings["globals"]["seeds_for_avg"]:
stgs_list_mlp.append(deepcopy(settings))
stgs_list_mlp[-1]["globals"]["seed"] = tmp_seed<split>
lreg_pred_train = lreg.predict(train_x)
 lreg_pred_train_rmse = evaluation_metric(lreg_pred_train, train_y)
 model_rmse_train.append(lreg_pred_train_rmse)
 lreg_pred_train_rmse
oof_pred_arr_avg_mlp = np.zeros(( len(X), 206))
test_pred_arr_avg_mlp = np.zeros(( len(X_test), 206))
for m_id, tmp_stgs in enumerate(stgs_list_mlp):
n_folds = tmp_stgs["globals"]["n_folds_split"]
tmp_seed = tmp_stgs["globals"]["seed"]
mlskf = MultilabelStratifiedKFold(n_splits=n_folds, shuffle=True, random_state=tmp_seed)
 train_val_indexs = list(mlskf.split(train_all_dataset.feat, train_all_dataset.label))
oof_pred_arr = np.zeros(( len(X), 206))
test_preds_arr = np.zeros(( n_folds, len(X_test), 206))
for fold_id in range(n_folds):
print("[fold: {} - model: {}]".format(fold_id, m_id))
model_path = ".. /input/moa-weight-stacking-5model-by-mlp-seed-cv/best_model_fold{}_model{}.pth".format(fold_id, m_id)
 model = MLP(**tmp_stgs["model"]["params"])
 model.load_state_dict(torch.load(model_path))
val_index = train_val_indexs[fold_id][1]
val_dataset = data.Subset(train_all_dataset, val_index)
 val_loader = data.DataLoader(val_dataset, **tmp_stgs["loader"]["val"])
 val_pred = inference_function(tmp_stgs, model, val_loader, device)
 oof_pred_arr[val_index] = val_pred
del val_dataset; del val_loader; del val_pred
test_loader = data.DataLoader(test_dataset, **tmp_stgs["loader"]["val"])
 test_pred = inference_function(tmp_stgs, model, test_loader, device)
 test_preds_arr[fold_id] = test_pred
del test_loader; del test_pred;
gc.collect()
oof_pred_arr_avg_mlp += oof_pred_arr
test_pred_arr_avg_mlp += test_preds_arr.mean(axis=0)
 oof_score = log_loss(y.ravel() , oof_pred_arr.ravel())
 cv_score_list_mlp.append([m_id, "oof", oof_score])
 oof_pred_arr_avg_mlp /= len(stgs_list_mlp)
 test_pred_arr_avg_mlp /= len(stgs_list_mlp )<create_dataframe>
start_time = time.time()
lreg_pred = lreg.predict(val_x)
 time_lapse = time.time() - start_time
model_pred_time.append(time_lapse )
pd.DataFrame(cv_score_list_mlp, columns=["model", "fold", "val loss"] )<compute_test_metric>
lreg_rmse = evaluation_metric(lreg_pred, val_y)
 model_rmse_val.append(lreg_rmse)
 lreg_rmse
print(log_loss(y.ravel() , oof_pred_arr_avg_mlp.ravel()))<init_hyperparams>
result = pd.DataFrame(
 {
'feature': model_type,
'train time': model_train_time,
'validation prediction time':model_pred_time,
'rmse train':model_rmse_train,
'rmse validation':model_rmse_val
} 
)
 result.sort_values(by='rmse validation' )
settings_str =
settings = yaml.safe_load(settings_str)
 settings["model"]["params"]["n_models"] = len(oof_list )<feature_engineering>
rfr=RandomForestRegressor(max_features=0.4, n_estimators=1000, random_state=RANDOM_STATE)
 start_time = time.time()
rfr.fit(train_x, train_y)
 time_lapse = time.time() - start_time
model_type.append('Random Forest')
 model_train_time.append(time_lapse)
 time_lapse
device = torch.device(settings["globals"]["device"])
 cv_score_list_cnn = []
stgs_list_cnn = []
for tmp_seed in settings["globals"]["seeds_for_avg"]:
stgs_list_cnn.append(deepcopy(settings))
stgs_list_cnn[-1]["globals"]["seed"] = tmp_seed<split>
rfr_pred_train = rfr.predict(train_x)
 rfr_pred_train_rmse = evaluation_metric(rfr_pred_train, train_y)
 model_rmse_train.append(rfr_pred_train_rmse)
 rfr_pred_train_rmse
oof_pred_arr_avg_cnn = np.zeros(( len(X), 206))
test_pred_arr_avg_cnn = np.zeros(( len(X_test), 206))
for m_id, tmp_stgs in enumerate(stgs_list_cnn):
n_folds = tmp_stgs["globals"]["n_folds_split"]
tmp_seed = tmp_stgs["globals"]["seed"]
mlskf = MultilabelStratifiedKFold(n_splits=n_folds, shuffle=True, random_state=tmp_seed)
 train_val_indexs = list(mlskf.split(train_all_dataset.feat, train_all_dataset.label))
oof_pred_arr = np.zeros(( len(X), 206))
test_preds_arr = np.zeros(( n_folds, len(X_test), 206))
train_all_dataset_2d.reset_model_order()
train_all_dataset_2d.shuffle_model_order(tmp_seed)
 test_dataset_2d.model_order = train_all_dataset_2d.model_order
print("[model {}'s order]: {}".format(m_id, train_all_dataset_2d.model_order))
for fold_id in range(n_folds):
print("[fold: {} - model: {}]".format(fold_id, m_id))
model_path = ".. /input/moa-weight-stacking-5model-by-1dcnn-seed-cv/best_model_cnn_fold{}_model{}.pth".format(fold_id, m_id)
 model = CNNForStacking(**tmp_stgs["model"]["params"])
 model.load_state_dict(torch.load(model_path))
val_index = train_val_indexs[fold_id][1]
val_dataset = data.Subset(train_all_dataset_2d, val_index)
 val_loader = data.DataLoader(val_dataset, **tmp_stgs["loader"]["val"])
 val_pred = inference_function(tmp_stgs, model, val_loader, device)
 oof_pred_arr[val_index] = val_pred
del val_dataset; del val_loader; del val_pred
test_loader = data.DataLoader(test_dataset_2d, **tmp_stgs["loader"]["val"])
 test_pred = inference_function(tmp_stgs, model, test_loader, device)
 test_preds_arr[fold_id] = test_pred
del test_loader; del test_pred;
gc.collect()
oof_pred_arr_avg_cnn += oof_pred_arr
test_pred_arr_avg_cnn += test_preds_arr.mean(axis=0)
 oof_score = log_loss(y.ravel() , oof_pred_arr.ravel())
 cv_score_list_cnn.append([m_id, "oof", oof_score])
 oof_pred_arr_avg_cnn /= len(stgs_list_cnn)
 test_pred_arr_avg_cnn /= len(stgs_list_cnn )<create_dataframe>
start_time = time.time()
rfr_pred_val = rfr.predict(val_x)
 time_lapse = time.time() - start_time
model_pred_time.append(time_lapse)
pd.DataFrame(cv_score_list_cnn, columns=["model", "fold", "val loss"] )<compute_test_metric>
rfr_rmse = evaluation_metric(rfr_pred_val, val_y)
 model_rmse_val.append(rfr_rmse)
 rfr_rmse
print(log_loss(y.ravel() , oof_pred_arr_avg_cnn.ravel()))<define_variables>
result = pd.DataFrame(
 {
'feature': model_type,
'train time': model_train_time,
'validation prediction time':model_pred_time,
'rmse train':model_rmse_train,
'rmse validation':model_rmse_val
} 
)
 result.sort_values(by='rmse validation' )
opt_weights = [ 0.1709929 , 0.22160569, 0.07207321, 0.95431566, -0.42108426]<compute_test_metric>
mnet = MLPRegressor(hidden_layer_sizes=(200,200,100, 100, 50, 25), activation = 'logistic', alpha = 0.001, max_iter = 1000, early_stopping = True, random_state = RANDOM_STATE)
 start_time = time.time()
mnet.fit(train_x, train_y)
 time_lapse = time.time() - start_time
model_type.append('Neural Net')
 model_train_time.append(time_lapse)
 time_lapse
oof_pred_arr_avg_wo = np.zeros(( len(X),206))
for w, oof_df in zip(opt_weights, oof_list):
oof_pred_arr_avg_wo += oof_df.values * w
print(log_loss(y.ravel() , oof_pred_arr_avg_wo.ravel()))<categorify>
mnet_pred_train = mnet.predict(train_x)
 mnet_pred_train_rmse = evaluation_metric(mnet_pred_train, train_y)
 model_rmse_train.append(mnet_pred_train_rmse)
 mnet_pred_train_rmse
BLEND_wo = sub_list[0].copy()
BLEND_wo.iloc[:, 1:] = 0.
 for w, sub_df in zip(opt_weights, sub_list):
BLEND_wo.iloc[:,1:] += sub_df.iloc[:,1:] * w<compute_test_metric>
start_time = time.time()
mnet_pred_val = mnet.predict(val_x)
 time_lapse = time.time() - start_time
model_pred_time.append(time_lapse )
oof_pred_arr_avg =(oof_pred_arr_avg_mlp + oof_pred_arr_avg_cnn + oof_pred_arr_avg_wo)/ 3
print(log_loss(y.ravel() , oof_pred_arr_avg.ravel()))<prepare_output>
mnet_rmse = evaluation_metric(mnet_pred_val, val_y)
 model_rmse_val.append(mnet_rmse)
 mnet_rmse
BLEND = sub_list[0].copy()
BLEND.iloc[:, 1:] =(BLEND_mlp.iloc[:, 1:] + BLEND_cnn.iloc[:, 1:] + BLEND_wo.iloc[:,1:])/ 3
BLEND.head()<load_from_csv>
result = pd.DataFrame(
 {
'feature': model_type,
'train time': model_train_time,
'validation prediction time':model_pred_time,
'rmse train':model_rmse_train,
'rmse validation':model_rmse_val
} 
)
 result.sort_values(by='rmse validation' )
if DO_SUBMISSION or DO_TRAIN_FOR_ENSEMBLE:
df_test = pd.read_csv(".. /input/lish-moa/test_features.csv")
 submission = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 elif DO_VIRTUAL_SUBMISSION:
df_test = pd.read_csv(".. /input/lish-moa/test_features.csv")
 submission = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 print(df_test.shape, submission.shape)
 df_test, submission = generate_virtual_private(df_test, submission)
 print(df_test.shape, submission.shape)
 else:
raise ValueError<load_from_csv>
params = {'n_estimators': 1000,
'max_depth': 5,
'min_samples_split': 6,
'learning_rate': 0.01,
'loss': 'ls',
'max_features':0.25,
'random_state':RANDOM_STATE}
gbm = GradientBoostingRegressor(**params)
 start_time = time.time()
gbm.fit(train_x, train_y)
 time_lapse = time.time() - start_time
model_type.append('Gradient Boosting')
 model_train_time.append(time_lapse)
 time_lapse
df = submission.copy()
public_id = list(df['sig_id'].values)
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = BLEND.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type =='ctl_vehicle'].sig_id] = 0
df_submit.to_csv('submission.csv',index=True )<set_options>
gbm_pred_train = gbm.predict(train_x)
 gbm_pred_train_rmse = evaluation_metric(gbm_pred_train, train_y)
 model_rmse_train.append(gbm_pred_train_rmse)
 gbm_pred_train_rmse
warnings.filterwarnings('ignore' )<install_modules>
start_time = time.time()
gbm_pred_val = gbm.predict(val_x)
 time_lapse = time.time() - start_time
model_pred_time.append(time_lapse )
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/ 
<load_from_csv>
gbm_rmse = evaluation_metric(gbm_pred_val, val_y)
 model_rmse_val.append(gbm_rmse)
 gbm_rmse
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 print('train_features: {}'.format(train_features.shape))
print('train_targets_scored: {}'.format(train_targets_scored.shape))
print('train_targets_nonscored: {}'.format(train_targets_nonscored.shape))
print('train_drug: {}'.format(train_drug.shape))
print('test_features: {}'.format(test_features.shape))
print('sample_submission: {}'.format(sample_submission.shape))<create_dataframe>
result = pd.DataFrame(
 {
'feature': model_type,
'train time': model_train_time,
'validation prediction time':model_pred_time,
'rmse train':model_rmse_train,
'rmse validation':model_rmse_val
} 
)
 result.sort_values(by='rmse validation' )
train_features2=train_features.copy()
test_features2=test_features.copy()<define_variables>
param = {'n_estimators': 1000,
'max_depth': 5,
'learning_rate': 0.01,
'random_state':RANDOM_STATE}
xgb = XGBRegressor(**param)
 start_time = time.time()
xgb.fit(train_x, train_y, eval_set=[(train_x, train_y),(val_x, val_y)], early_stopping_rounds = 5, eval_metric = 'rmse')
 time_lapse = time.time() - start_time
model_type.append('XGBoost')
 model_train_time.append(time_lapse)
 time_lapse
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<create_dataframe>
xgb_pred_train = xgb.predict(train_x)
 xgb_pred_train_rmse = evaluation_metric(xgb_pred_train, train_y)
 model_rmse_train.append(xgb_pred_train_rmse)
 xgb_pred_train_rmse
qt = QuantileTransformer(n_quantiles=100,random_state=42,output_distribution='normal')
 data = pd.concat([pd.DataFrame(train_features[GENES+CELLS]), pd.DataFrame(test_features[GENES+CELLS])])
 data2 = qt.fit_transform(data[GENES+CELLS])
 train_features[GENES+CELLS] = pd.DataFrame(data2[:train_features.shape[0]])
 test_features[GENES+CELLS] = pd.DataFrame(data2[-test_features.shape[0]:] )<set_options>
start_time = time.time()
xgb_pred_val = xgb.predict(val_x)
 time_lapse = time.time() - start_time
model_pred_time.append(time_lapse )
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )<create_dataframe>
xgb_rmse = evaluation_metric(xgb_pred_val, val_y)
 model_rmse_val.append(xgb_rmse)
 xgb_rmse
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1 )<create_dataframe>
result = pd.DataFrame(
 {
'feature': model_type,
'train time': model_train_time,
'validation prediction time':model_pred_time,
'rmse train':model_rmse_train,
'rmse validation':model_rmse_val
} 
)
 result.sort_values(by='rmse validation' )
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1 )<create_dataframe>
test_pred = gbm.predict(test_set)
 test_result = pd.DataFrame(
 {
'Id': test_set_Id,
'SalePrice': np.exp(test_pred),
} )
var_thresh = VarianceThreshold(0.85)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
<concatenate>
test_result.to_csv('submission_gbm.csv', index=False )
def fe_cluster(train, test, n_clusters_g = 22, n_clusters_c = 4, SEED = 42):
features_g = GENES
features_c = CELLS
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster(train_features2,test_features2 )<concatenate>
p_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 p_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )<categorify>
def create_new_columns(dataset):
dataset['total_square_footage'] = dataset['TotalBsmtSF'] + dataset['GrLivArea']
dataset['remodelling_done'] = dataset.apply(lambda x: verify_remodelling(x['YearRemodAdd'], x['YearBuilt']), axis=1)
 dataset['selling_season'] = dataset.apply(lambda x: selling_period(x['MoSold']), axis=1)
 dataset['total_area_of_house'] = dataset['TotalBsmtSF'] + dataset['1stFlrSF'] + dataset['2ndFlrSF']
dataset['age_of_building'] = dataset['YearBuilt'].apply(lambda x: pd.datetime.now().year - x)
 return dataset
def map_ordinal_features(dataset):
for col in ordinal_cat_features:
dataset[col] = dataset[col].map(ordinal_ranking)
 dataset[col] = dataset[col].fillna(0)
 return dataset
def log_continious(dataset):
imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')
 for col in continious_features:
imp_mean.fit(dataset[col].values.reshape(-1,1))
dataset[col] = imp_mean.transform(dataset[col].values.reshape(-1,1))
dataset[col] = np.log(dataset[col] + 1)
 return dataset
def nominal_one_hot(dataset):
for col in norminal_features:
dataset[col] = dataset[col].astype(str)
 dataset[col] = dataset[col].fillna('Unknown')
 return dataset
def discrete_fill(dataset):
for col in discrete_features:
dataset[col] = dataset[col].fillna(0)
 return dataset
def target_feature(train):
target = np.log(train['SalePrice'] + 1)
 return target
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_pca'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_pca'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )<split>
target = target_feature(p_train )
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]<split>
pipes = Pipeline(steps = [('new_columns', FunctionTransformer(create_new_columns)) , 
('continious_cleanup', FunctionTransformer(log_continious)) , 
('nominal_filling', FunctionTransformer(nominal_one_hot)) , 
('discrete_filling',FunctionTransformer(discrete_fill)) , 
('ordinal_cleanup', FunctionTransformer(map_ordinal_features)) ,
])
 pipes.fit(p_train )
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]<define_variables>
p_train = pipes.transform(p_train )
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']<feature_engineering>
p_test = pipes.transform(p_test )
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )<split>
p_test_Id = p_test['Id']
p_train = p_train.drop(columns=['SalePrice', 'Id'])
 p_test = p_test.drop(columns = 'Id')
 p_train_x, p_val_x, p_train_y, p_val_y = train_test_split(p_train, target, train_size=0.8, test_size=0.2, random_state = RANDOM_STATE )
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:] 
<concatenate>
categorical_transformer = OneHotEncoder(handle_unknown = "ignore", sparse=False)
 numeric_transformer = Pipeline(steps=[ 
('scaler', StandardScaler())])
 preprocessor = ColumnTransformer(remainder='passthrough',
transformers=[ 
('num', numeric_transformer, selector(dtype_exclude="object")) , 
('cat', categorical_transformer, selector(dtype_include="object"))
] )
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )<merge>
params = {'n_estimators': 1000,
'max_depth': 5,
'min_samples_split': 6,
'learning_rate': 0.01,
'loss': 'ls',
'max_features':0.25,
'random_state':RANDOM_STATE}
model_gbm = GradientBoostingRegressor(**params)
 gbm_pipeline = Pipeline(steps=[ 
('preprocess', preprocessor), 
('model', model_gbm)
 ])
 gbm_pipeline.fit(p_train_x, p_train_y)
 gbm_preds = gbm_pipeline.predict(p_val_x)
 score = evaluation_metric(gbm_preds, p_val_y)
 print('RMSE:', score )
<drop_column><EOS> 
p_test_gbm = gbm_pipeline.predict(p_test)
 test_result = pd.DataFrame(
 {
'Id': p_test_Id,
'SalePrice': np.exp(p_test_gbm),
})
 test_result.to_csv('pipeline_gbm.csv', index=False)