Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<count_values>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
import torch
import torch.nn as nn
from torch.utils.data import Dataset,DataLoader
from sklearn.preprocessing import QuantileTransformer
from sklearn import preprocessing
import torch.optim as optim
print(X_train["Outlier"].value_counts())
 print(X_test["Outlier"].value_counts() )<features_selection>
import torch.nn.functional as F
model_select = SelectFromModel(Lasso(alpha=0.0005,tol=0.001))
ms = model_select.fit(X_train,y_train)
 selected_feat = X_train.columns[(ms.get_support())]
print('total features: {}'.format(( X_train.shape[1])))
 print('selected features: {}'.format(len(selected_feat)))
 print('features with coefficients shrank to zero: {}'.format(
 np.sum(ms.estimator_.coef_ == 0)))
 shrink_zero_num=np.sum(ms.estimator_.coef_ == 0)
 print("Number of features which coefficient was shrank to zero: ",shrink_zero_num)
 removed_feats = X_train.columns[(ms.estimator_.coef_ == 0 ).ravel().tolist() ]
print("The removed features: ",removed_feats )<train_model>
import optuna
import plotly as pl
rf = RandomForestRegressor(n_estimators=200)
 rf = rf.fit(X_train,y_train)
 rf_importance=pd.DataFrame({'Features':np.array(X_train.columns),'Importance':rf.feature_importances_*100 } ).sort_values(by='Importance', ascending=False)
 rf_importance<features_selection>
from sklearn.model_selection import StratifiedKFold
model = SelectFromModel(rf, prefit=True,threshold=0.0005)
 columns_select_rf=X_train.columns[model.get_support() ]
print("kept {} features after tree-based feature selection".format(len(columns_select_rf)) ,"
The features are : ",columns_select_rf )<normalization>
import lightgbm as lgb
def select_features(X_train, y_train, X_test):
fs = SelectKBest(score_func=f_regression, k='all')
 fs.fit(X_train, y_train)
 X_train_fs = fs.transform(X_train)
 X_test_fs = fs.transform(X_test)
 return X_train_fs, X_test_fs, fs
X_train_fs, X_test_fs, fs = select_features(X_train, y_train, X_test)
 clf_importance=pd.DataFrame({'Features':np.array(X_train.columns),'Importance':fs.scores_ } ).sort_values(by='Importance', ascending=False)
 clf_importance<normalization>
device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
def select_features(X_train, y_train, X_test):
fs = SelectKBest(score_func=mutual_info_regression, k='all')
 fs.fit(X_train, y_train)
 X_train_fs = fs.transform(X_train)
 X_test_fs = fs.transform(X_test)
 return X_train_fs, X_test_fs, fs
X_train_fs, X_test_fs, fs = select_features(X_train, y_train, X_test)
 clf_importance=pd.DataFrame({'Features':np.array(X_train.columns),'Importance':fs.scores_ } ).sort_values(by='Importance', ascending=False)
 clf_importance<drop_column>
test_fea = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_fea = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_tar_nonsco = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_tar_sco = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
X_train=X_train[columns_select_mul]
X_test=X_test[columns_select_mul]<predict_on_test>
def seed_everything(seed=1062):
np.random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=1062 )
kmeans = KMeans(n_clusters=3, random_state=25 ).fit(X_train)
 X_train['cluster']=kmeans.labels_
X_test['cluster']=kmeans.predict(X_test )<count_values>
class Sparsemax(nn.Module):
def __init__(self, dim=None):
super(Sparsemax, self ).__init__()
self.dim = -1 if dim is None else dim
def forward(self, input):
input = input.transpose(0, self.dim)
 original_size = input.size()
input = input.reshape(input.size(0), -1)
 input = input.transpose(0, 1)
 dim = 1
number_of_logits = input.size(dim)
 input = input - torch.max(input, dim=dim, keepdim=True)[0].expand_as(input)
 zs = torch.sort(input=input, dim=dim, descending=True)[0]
range = torch.arange(start=1, end=number_of_logits + 1, device=device,step=1, dtype=input.dtype ).view(1, -1)
 range = range.expand_as(zs)
 bound = 1 + range * zs
cumulative_sum_zs = torch.cumsum(zs, dim)
 is_gt = torch.gt(bound, cumulative_sum_zs ).type(input.type())
 k = torch.max(is_gt * range, dim, keepdim=True)[0]
zs_sparse = is_gt * zs
taus =(torch.sum(zs_sparse, dim, keepdim=True)- 1)/ k
taus = taus.expand_as(input)
 self.output = torch.max(torch.zeros_like(input), input - taus)
 output = self.output
output = output.transpose(0, 1)
 output = output.reshape(original_size)
 output = output.transpose(0, self.dim)
 return output
def backward(self, grad_output):
dim = 1
nonzeros = torch.ne(self.output, 0)
 sum = torch.sum(grad_output * nonzeros, dim=dim)/ torch.sum(nonzeros, dim=dim)
 self.grad_input = nonzeros *(grad_output - sum.expand_as(grad_output))
return self.grad_input
pd.DataFrame(kmeans.labels_ ).value_counts()<categorify>
def initialize_non_glu(module,inp_dim,out_dim):
gain = np.sqrt(( inp_dim+out_dim)/np.sqrt(4*inp_dim))
torch.nn.init.xavier_normal_(module.weight, gain=gain)
 class GBN(nn.Module):
def __init__(self,inp,vbs=128,momentum=0.01):
super().__init__()
self.bn = nn.BatchNorm1d(inp,momentum=momentum)
 self.vbs = vbs
def forward(self,x):
chunk = torch.chunk(x,max(1,x.size(0)//self.vbs),0)
 res = [self.bn(y)for y in chunk ]
return torch.cat(res,0)
 class GLU(nn.Module):
def __init__(self,inp_dim,out_dim,fc=None,vbs=128):
super().__init__()
if fc:
self.fc = fc
else:
self.fc = nn.Linear(inp_dim,out_dim*2)
 self.bn = GBN(out_dim*2,vbs=vbs)
 self.od = out_dim
def forward(self,x):
x = self.bn(self.fc(x))
return x[:,:self.od]*torch.sigmoid(x[:,self.od:])
 class FeatureTransformer(nn.Module):
def __init__(self,inp_dim,out_dim,shared,n_ind,vbs=128):
super().__init__()
first = True
self.shared = nn.ModuleList()
if shared:
self.shared.append(GLU(inp_dim,out_dim,shared[0],vbs=vbs))
first= False
for fc in shared[1:]:
self.shared.append(GLU(out_dim,out_dim,fc,vbs=vbs))
else:
self.shared = None
self.independ = nn.ModuleList()
if first:
self.independ.append(GLU(inp,out_dim,vbs=vbs))
for x in range(first, n_ind):
self.independ.append(GLU(out_dim,out_dim,vbs=vbs))
self.scale = torch.sqrt(torch.tensor([.5],device=device))
def forward(self,x):
if self.shared:
x = self.shared[0](x)
 for glu in self.shared[1:]:
x = torch.add(x, glu(x))
x = x*self.scale
for glu in self.independ:
x = torch.add(x, glu(x))
x = x*self.scale
return x
class AttentionTransformer(nn.Module):
def __init__(self,inp_dim,out_dim,relax,vbs=128):
super().__init__()
self.fc = nn.Linear(inp_dim,out_dim)
 self.bn = GBN(out_dim,vbs=vbs)
 self.r = torch.tensor([relax],device=device)
 def forward(self,a,priors):
a = self.bn(self.fc(a))
mask = torch.sigmoid(a*priors)
 priors =priors*(self.r-mask)
 return mask
class DecisionStep(nn.Module):
def __init__(self,inp_dim,n_d,n_a,shared,n_ind,relax,vbs=128):
super().__init__()
self.fea_tran = FeatureTransformer(inp_dim,n_d+n_a,shared,n_ind,vbs)
 self.atten_tran = AttentionTransformer(n_a,inp_dim,relax,vbs)
 def forward(self,x,a,priors):
mask = self.atten_tran(a,priors)
 loss =(( -1)*mask*torch.log(mask+1e-10)).mean()
x = self.fea_tran(x*mask)
 return x,loss
class TabNet(nn.Module):
def __init__(self,inp_dim,final_out_dim,n_d=64,n_a=64,n_shared=2,n_ind=2,n_steps=5,relax=1.2,vbs=128):
super().__init__()
if n_shared>0:
self.shared = nn.ModuleList()
self.shared.append(nn.Linear(inp_dim,2*(n_d+n_a)))
 for x in range(n_shared-1):
self.shared.append(nn.Linear(n_d+n_a,2*(n_d+n_a)))
 else:
self.shared=None
self.first_step = FeatureTransformer(inp_dim,n_d+n_a,self.shared,n_ind)
 self.steps = nn.ModuleList()
for x in range(n_steps-1):
self.steps.append(DecisionStep(inp_dim,n_d,n_a,self.shared,n_ind,relax,vbs))
self.fc = nn.Linear(n_d,final_out_dim)
 self.bn = nn.BatchNorm1d(inp_dim)
 self.n_d = n_d
def forward(self,x):
x = self.bn(x)
 x_a = self.first_step(x)[:,self.n_d:]
loss = torch.zeros(1 ).to(x.device)
 out = torch.zeros(x.size(0),self.n_d ).to(x.device)
 priors = torch.ones(x.shape ).to(x.device)
 for step in self.steps:
x_te,l = step(x,x_a,priors)
 out += F.relu(x_te[:,:self.n_d])
 x_a = x_te[:,self.n_d:]
loss += l
return self.fc(out),loss
X_train['cluster']=X_train['cluster'].astype(str)
 X_test['cluster']=X_test['cluster'].astype(str)
 X_train = pd.get_dummies(X_train)
 X_test = pd.get_dummies(X_test)
 X_train, X_test = X_train.align(X_test, join='left', axis=1, fill_value=0 )<drop_column>
class TabNetWithEmbed(nn.Module):
def __init__(self,inp_dim,final_out_dim,n_d=64,n_a=64,n_shared=2,n_ind=2,n_steps=5,relax=1.2,vbs=128):
super().__init__()
self.tabnet = TabNet(inp_dim,final_out_dim,n_d,n_a,n_shared,n_ind,n_steps,relax,vbs)
 self.cat_embed = []
self.emb1 = nn.Embedding(2,1)
 self.emb3 = nn.Embedding(3,1)
 self.cat_embed.append(self.emb1)
 self.cat_embed.append(self.emb3)
 def forward(self,catv,contv):
catv = catv.to(device)
 contv = contv.to(device)
 embeddings = [embed(catv[:,idx])for embed,idx in zip(self.cat_embed,range(catv.size(1)))]
catv = torch.cat(embeddings,1)
 x = torch.cat(( catv,contv),1 ).contiguous()
x,l = self.tabnet(x)
 return torch.sigmoid(x),l
X_train=X_train.drop(X_train[to_drop], axis=1)
 X_test=X_test.drop(X_test[to_drop], axis=1 )<split>
class DrugData(Dataset):
def __init__(self,df,out,train=True):
self.df = df
self.out = out
self.train=train
def __len__(self):
return len(self.df)
 def __getitem__(self,idx):
if self.train:
tar = np.where(self.out[idx].reshape(-1)==0,0.00001,0.99999)
 return torch.from_numpy(self.df[idx,:] ).float() ,torch.tensor(tar ).float()
else:
return torch.from_numpy(self.df[idx,:] ).float() ,torch.tensor(self.out[idx].reshape(-1)).float()
X_tra,X_val,y_tra,y_val = train_test_split(X_train, y_train, test_size = 0.25, random_state = 25)
 print(X_tra.shape)
 print(X_val.shape)
 print(y_tra.shape)
 print(y_val.shape )<compute_train_metric>
train_fea['cp_dose'].replace({'D1':0,'D2':1},inplace=True)
 test_fea['cp_dose'].replace({'D1':0,'D2':1},inplace=True)
 train_fea['cp_time'].replace({24:0,72:1,48:2},inplace=True)
 test_fea['cp_time'].replace({24:0,72:1,48:2},inplace=True)
 train_fea['cp_type'].replace({'trt_cp':0,'ctl_vehicle':1},inplace=True)
 test_fea['cp_type'].replace({'trt_cp':0,'ctl_vehicle':1},inplace=True)
 train_fea = train_fea.drop(columns='sig_id')
 test_fea = test_fea.drop(columns='sig_id' )
start_time = time.time()
regr = make_pipeline(RobustScaler() ,linear_model.LinearRegression())
 regr.fit(X_tra,y_tra)
 y_tra_regr_pred=regr.predict(X_tra)
 print("The MSE on train set:",np.sqrt(metrics.mean_squared_error(y_tra, y_tra_regr_pred)))
 y_val_regr_pred = regr.predict(X_val)
 print("The MSE on validation set:",np.sqrt(metrics.mean_squared_error(y_val, y_val_regr_pred)))
 regr_mse = cross_val_score(regr , X_train, y_train, scoring='neg_mean_squared_error',cv=5)
 print("MSE on Cross-validation:
",np.sqrt(-regr_mse))
print("The mean MSE on Cross-validation is {:.3f}".format(np.mean(np.sqrt(-regr_mse))))
print("--- %s seconds ---" %(time.time() - start_time))
plt.figure('regr')
 plt.plot([np.expm1(y_val.min()),np.expm1(y_val.max())],[np.expm1(y_val.min()),np.expm1(y_val.max())],'k--',lw=2)
 plt.scatter(np.expm1(y_val),np.expm1(y_val_regr_pred))
plt.xlabel('Fact Price')
 plt.ylabel('Predict Price')
 plt.title('Predict Effect Using Linear Regression')
 plt.show()<compute_train_metric>
cat_col = [0,2]
num_col = len(train_fea.columns )
rcv = RidgeCV(alphas=np.arange(3.0,6.0,0.1))
rcv.fit(X_tra,y_tra)
 print(rcv.alpha_)
 y_tra_rcv_pred=rcv.predict(X_tra)
 print("使用Ridge Regression cv模型的均方根误差为(train):",np.sqrt(metrics.mean_squared_error(y_tra, y_tra_rcv_pred)))
 y_rcv_pred = rcv.predict(X_val)
 print("使用Ridge Regression cv模型的均方根误差为(Validation):",np.sqrt(metrics.mean_squared_error(y_val, y_rcv_pred)) )<compute_train_metric>
train_tar = train_tar_sco.drop(columns='sig_id' ).values
start_time = time.time()
regr_ridge= make_pipeline(RobustScaler() , linear_model.Ridge(alpha = 4.7))
regr_ridge.fit(X_tra,y_tra)
 y_tra_ridge_pred=regr_ridge.predict(X_tra)
 print("The MSE on train set:",np.sqrt(metrics.mean_squared_error(y_tra,y_tra_ridge_pred)))
 y_val_ridge_pred = regr_ridge.predict(X_val)
 print("The MSE on validation set:",np.sqrt(metrics.mean_squared_error(y_val, y_val_ridge_pred)))
 regr_mse = cross_val_score(regr_ridge, X_train, y_train, scoring='neg_mean_squared_error',cv=5)
 print("MSE on Cross-validation:
",np.sqrt(-regr_mse))
print("The mean MSE on Cross-validation is {:.3f}".format(np.mean(np.sqrt(-regr_mse))))
print("--- %s seconds ---" %(time.time() - start_time))
plt.figure('ridge_regr')
 plt.plot([np.expm1(y_val.min()),np.expm1(y_val.max())],[np.expm1(y_val.min()),np.expm1(y_val.max())],'k--',lw=2)
 plt.scatter(np.expm1(y_val),np.expm1(y_val_ridge_pred))
plt.xlabel('Fact Price')
 plt.ylabel('Predict Price')
 plt.title('Predict Effect Using Ridge Regression')
 plt.show()<train_on_grid>
from sklearn.feature_selection import VarianceThreshold
from sklearn.decomposition import PCA
lscv = LassoCV(alphas=np.arange(0.0001,0.0005,0.00001))
lscv.fit(X_tra,y_tra.values.ravel())
 print('Best parameters found by randomised search:', lscv.alpha_, '
' )<compute_train_metric>
data = pd.concat([train_fea,test_fea],ignore_index=True)
 g = [*(x for x in data.columns if 'g' in x)]
c = [*(x for x in data.columns if 'c-' in x)]
start_time = time.time()
lasso= make_pipeline(RobustScaler() ,linear_model.Lasso(alpha=0.00043999999999999996))
lasso.fit(X_tra,y_tra)
 y_tra_lasso_pred=lasso.predict(X_tra)
 print("The MSE on train set:",np.sqrt(metrics.mean_squared_error(y_tra,y_tra_lasso_pred)))
 y_val_lasso_pred = lasso.predict(X_val)
 print("The MSE on validation set:",np.sqrt(metrics.mean_squared_error(y_val, y_val_lasso_pred)))
 regr_mse = cross_val_score(lasso, X_train, y_train, scoring='neg_mean_squared_error',cv=5)
 print("MSE on Cross-validation:
",np.sqrt(-regr_mse))
print("The mean MSE on Cross-validation is {:.3f}".format(np.mean(np.sqrt(-regr_mse))))
print("--- %s seconds ---" %(time.time() - start_time))
plt.figure('ridge_regr')
 plt.plot([np.expm1(y_val.min()),np.expm1(y_val.max())],[np.expm1(y_val.min()),np.expm1(y_val.max())],'k--',lw=2)
 plt.scatter(np.expm1(y_val),np.expm1(y_val_lasso_pred))
plt.xlabel('Fact Price')
 plt.ylabel('Predict Price')
 plt.title('Predict Effect Using Lasso Regression')
 plt.show()<choose_model_class>
for col in g:
sel = QuantileTransformer(n_quantiles=1000,random_state=0,output_distribution='normal')
 sel.fit(data[col].to_numpy().reshape(-1,1))
data[col] = sel.transform(data[col].to_numpy().reshape(-1,1))
for col in c:
sel = QuantileTransformer(n_quantiles=1000,random_state=0,output_distribution='normal')
 sel.fit(data[col].to_numpy().reshape(-1,1))
data[col] = sel.transform(data[col].to_numpy().reshape(-1,1))
start_time = time.time()
forest = make_pipeline(RobustScaler() , RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,
max_features=0.95, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=3, min_samples_split=5,
min_weight_fraction_leaf=0.0, n_estimators=100,
n_jobs=None, oob_score=True, random_state=23,
verbose=0, warm_start=False))
forest.fit(X_tra,y_tra.values.ravel())
 y_tra_forest_pred=forest.predict(X_tra)
 print("The MSE on train set:",np.sqrt(metrics.mean_squared_error(y_tra, y_tra_forest_pred)))
 y_val_forest_pred = forest.predict(X_val)
 print("The MSE on validation set:",np.sqrt(metrics.mean_squared_error(y_val, y_val_forest_pred)))
 regr_mse = cross_val_score(forest, X_train, y_train, scoring='neg_mean_squared_error',cv=5)
 print("MSE on Cross-validation:
",np.sqrt(-regr_mse))
print("The mean MSE on Cross-validation is {:.3f}".format(np.mean(np.sqrt(-regr_mse))))
print("--- %s seconds ---" %(time.time() - start_time))
plt.figure('forest')
 plt.plot([np.expm1(y_val.min()),np.expm1(y_val.max())],[np.expm1(y_val.min()),np.expm1(y_val.max())],'k--',lw=2)
 plt.scatter(np.expm1(y_val),np.expm1(y_val_forest_pred))
plt.xlabel('Fact Price')
 plt.ylabel('Predict Price')
 plt.title('Predict Effect Using RandomForest')
 plt.show() 
<choose_model_class>
pca_c = PCA(n_components=15)
 extra_c = pd.DataFrame(pca_c.fit_transform(data[c]))
pca_g = PCA(n_components=50)
 extra_g = pd.DataFrame(pca_g.fit_transform(data[g]))
data = pd.concat(( data,extra_c,extra_g),axis=1 )
start_time = time.time()
xgb = make_pipeline(RobustScaler() ,XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.82, gamma=0,
importance_type='gain', learning_rate=0.2,
max_delta_step=0, max_depth=5, min_child_weight=7, missing=None,
n_estimators=200, n_jobs=-1, nthread=None, objective='reg:squarederror',
random_state=3, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,
seed=None, silent=None, subsample=0.7526315789473684, verbosity=1))
xgb.fit(X_tra,y_tra.values.ravel())
 print("Validation:", xgb.score(X_val,y_val))
y_tra_xgb_pred=xgb.predict(X_tra)
 print("The MSE on train set:",np.sqrt(metrics.mean_squared_error(y_tra, y_tra_xgb_pred)))
 y_val_xgb_pred = xgb.predict(X_val)
 print("The MSE on validation set:",np.sqrt(metrics.mean_squared_error(y_val, y_val_xgb_pred)))
 regr_mse = cross_val_score(xgb, X_train, y_train, scoring='neg_mean_squared_error',cv=5)
 print("MSE on Cross-validation:
",np.sqrt(-regr_mse))
print("The mean MSE on Cross-validation is {:.3f}".format(np.mean(np.sqrt(-regr_mse))))
print("--- %s seconds ---" %(time.time() - start_time))
plt.figure('XGBoost')
 plt.plot([np.expm1(y_val.min()),np.expm1(y_val.max())],[np.expm1(y_val.min()),np.expm1(y_val.max())],'k--',lw=2)
 plt.scatter(np.expm1(y_val),np.expm1(y_val_xgb_pred))
plt.xlabel('Fact Price')
 plt.ylabel('Predict Price')
 plt.title('Predict Effect Using XGBoost')
 plt.show()<choose_model_class>
from umap import UMAP
start_time = time.time()
lgb = LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 lgb.fit(X_tra,y_tra.values.ravel())
 print("Validation:", lgb.score(X_val,y_val))
y_tra_lgb_pred=lgb.predict(X_tra)
 print("The MSE on train set:",np.sqrt(metrics.mean_squared_error(y_tra, y_tra_lgb_pred)))
 y_val_lgb_pred = lgb.predict(X_val)
 print("The MSE on validation set:",np.sqrt(metrics.mean_squared_error(y_val, y_val_lgb_pred)))
 regr_mse = cross_val_score(lgb, X_train, y_train, scoring='neg_mean_squared_error',cv=5)
 print("MSE on Cross-validation:
",np.sqrt(-regr_mse))
print("The mean MSE on Cross-validation is {:.3f}".format(np.mean(np.sqrt(-regr_mse))))
print("--- %s seconds ---" %(time.time() - start_time))
plt.figure('lgb')
 plt.plot([np.expm1(y_val.min()),np.expm1(y_val.max())],[np.expm1(y_val.min()),np.expm1(y_val.max())],'k--',lw=2)
 plt.scatter(np.expm1(y_val),np.expm1(y_val_lgb_pred))
plt.xlabel('Fact Price')
 plt.ylabel('Predict Price')
 plt.title('Predict Effect Using LightGBM')
 plt.show()<compute_train_metric>
umap_c = UMAP(random_state=256,n_components=15)
 extra_c = pd.DataFrame(umap_c.fit_transform(data[c]))
umap_g = UMAP(random_state=256,n_components=50)
 extra_g = pd.DataFrame(umap_g.fit_transform(data[g]))
data = pd.concat(( data,extra_c,extra_g),axis=1 )
start_time = time.time()
svr = SVR(kernel = 'poly')
 svr.fit(X_tra,y_tra.values.ravel())
 print("Validation:", svr.score(X_val,y_val))
y_tra_svr_pred=svr.predict(X_tra)
 print("The MSE on train set:",np.sqrt(metrics.mean_squared_error(y_tra, y_tra_svr_pred)))
 y_val_svr_pred = svr.predict(X_val)
 print("The MSE on validation set:",np.sqrt(metrics.mean_squared_error(y_val, y_val_svr_pred)))
 regr_mse = cross_val_score(svr, X_train, y_train, scoring='neg_mean_squared_error',cv=5)
 print("MSE on Cross-validation:
",np.sqrt(-regr_mse))
print("The mean MSE on Cross-validation is {:.3f}".format(np.mean(np.sqrt(-regr_mse))))
print("--- %s seconds ---" %(time.time() - start_time))
plt.figure('svr')
 plt.plot([np.expm1(y_val.min()),np.expm1(y_val.max())],[np.expm1(y_val.min()),np.expm1(y_val.max())],'k--',lw=2)
 plt.scatter(np.expm1(y_val),np.expm1(y_val_svr_pred))
plt.xlabel('Fact Price')
 plt.ylabel('Predict Price')
 plt.title('Predict Effect Using SVR')
 plt.show()<predict_on_test>
train_df = data.iloc[:len(train_fea),:].values
test_df= data.iloc[len(train_fea):,:].values
def stacked(all_data):
stacked_pred=0.1*regr.predict(all_data)+0.1*regr_ridge.predict(all_data)+\
0.1*lasso.predict(all_data)+0.1*forest.predict(all_data)+0.2*xgb.predict(all_data)+\
0.2*lgb.predict(all_data)+0.2*svr.predict(all_data)
 return stacked_pred<concatenate>
kfold = StratifiedKFold(n_splits=20 )
Prediction_final=np.expm1(stacked(X_test))<save_to_csv>
loss_func = nn.BCELoss()
sub = pd.DataFrame()
sub['Id'] = test_data['Id']
sub['SalePrice'] = Prediction_final
print(sub.head())
 sub.to_csv('submission.csv',index=False )<load_from_csv>
ra = np.arange(3,train_df.shape[1])
 np.random.shuffle(ra)
 train_df[:,3:] = train_df[:,ra]
test_df[:,3:] = test_df[:,ra]
train_df = pd.read_csv('.. /input/train.csv')
 print('Rows: ',train_df.shape[0],'Columns: ',train_df.shape[1])
 train_df.info()<count_values>
cat_col = [0,1]
train_df['target'].value_counts()<load_from_csv>
seed_everything(1006)
 submission.iloc[:,1:]=0
for train,test in kfold.split(train_df,np.zeros(len(train_df))):
batch_size=512
sparse_constant = 0
model = TabNetWithEmbed(train_df.shape[1]-1,train_tar.shape[1],n_d=128,n_a=16,n_shared=1,n_ind=4,n_steps=3,relax=1.5,vbs=64)
 model.to(device)
 torch.cuda.empty_cache()
optimizer = optim.Adam(model.parameters() ,lr=0.007809719000164987,weight_decay=0.00001)
 sched = optim.lr_scheduler.ReduceLROnPlateau(optimizer,factor=0.1,patience=3,verbose=True)
 train_dataset = DrugData(train_df[train],train_tar[train])
 valid_dataset = DrugData(train_df[test],train_tar[test],False)
 train_loader = DataLoader(train_dataset,batch_size=batch_size,num_workers=4,shuffle=True)
 valid_loader = DataLoader(valid_dataset,batch_size=batch_size,num_workers=4,shuffle=True)
 losses=[]
norm = []
t = time.time()
for x in range(24):
train_loss=0.
 grad_norm_sum = 0.
 for inp,tar in train_loader:
model.zero_grad()
out,l = model(inp[:,cat_col].long() ,inp[:,3:])
 loss = loss_func(out,tar.to(device))
loss.backward()
optimizer.step()
train_loss+=loss.item() *tar.size(0)
 valid_loss=0.
 v=0
for inp,tar in valid_loader:
v+=1
out,_ = model(inp[:,cat_col].long() ,inp[:,3:])
 loss = loss_func(out,tar.to(device))
valid_loss+= loss.item() *tar.size(0)
 losses.append(valid_loss/len(valid_dataset))
print('%d epoch, %.8f valid_loss, %.8f training_loss %fsec time'%(x+1,losses[-1],train_loss/len(train_dataset),(time.time() -t)))
 sched.step(losses[-1])
 t = time.time()
print("completed training one fold -------------- ")
 model.eval()
submission.iloc[:,1:] += model(torch.from_numpy(test_df[:,cat_col] ).long() ,torch.from_numpy(test_df[:,3:] ).float())[0].data.cpu().numpy()
test_df = pd.read_csv('.. /input/test.csv' )<drop_column>
X_test = test_df.drop('ID_code',axis=1 )<prepare_x_and_y>
X = train_df.drop(['ID_code','target'],axis=1)
 y = train_df['target']<choose_model_class>
submission.iloc[:,1:] = submission.iloc[:,1:]/20
n_fold = 5
folds = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=42 )<init_hyperparams>
submission = submission.copy()
<train_model><EOS> 
submission.to_csv('submission.csv',index=False )