Kaggle competition: House Prices - Advanced Regression Techniques
train_reg['age'] =(train_reg['Age'] - train_reg['Age'].min())/(train_reg['Age'].max() - train_reg['Age'].min())
 train_reg['BASE'] =(train_reg['min_FVC'] - train_reg['min_FVC'].min())/(train_reg['min_FVC'].max() - train_reg['min_FVC'].min())
 train_reg['percent'] =(train_reg['min_Percent'] - train_reg['min_Percent'].min())/(train_reg['min_Percent'].max() - train_reg['min_Percent'].min())
 train_reg['volume'] =(train_reg['Volume'] - train_reg['Volume'].min())/(train_reg['Volume'].max() - train_reg['Volume'].min())
 train_reg['mean'] =(train_reg['Mean'] - train_reg['Mean'].min())/(train_reg['Mean'].max() - train_reg['Mean'].min())
 train_reg['skew'] =(train_reg['Skew'] - train_reg['Skew'].min())/(train_reg['Skew'].max() - train_reg['Skew'].min())
 train_reg['median'] =(train_reg['Median'] - train_reg['Median'].min())/(train_reg['Median'].max() - train_reg['Median'].min())
 train_reg['kurthosis'] =(train_reg['Kurthosis'] - train_reg['Kurthosis'].min())/(train_reg['Kurthosis'].max() - train_reg['Kurthosis'].min())
 train_reg['haa'] =(train_reg['HAA'] - train_reg['HAA'].min())/(train_reg['HAA'].max() - train_reg['HAA'].min())
 train_reg['midmedian'] =(train_reg['midMedian'] - train_reg['midMedian'].min())/(train_reg['midMedian'].max() - train_reg['midMedian'].min())
 train_reg['midmean'] =(train_reg['midMean'] - train_reg['midMean'].min())/(train_reg['midMean'].max() - train_reg['midMean'].min())
 train_reg['midskew'] =(train_reg['midSkew'] - train_reg['midSkew'].min())/(train_reg['midSkew'].max() - train_reg['midSkew'].min())
 train_reg['midkurthosis'] =(train_reg['midKurthosis'] - train_reg['midKurthosis'].min())/(train_reg['midKurthosis'].max() - train_reg['midKurthosis'].min())
 train_reg['midhaa'] =(train_reg['midHAA'] - train_reg['midHAA'].min())/(train_reg['midHAA'].max() - train_reg['midHAA'].min())
 train_reg['pro_0'] =(train_reg['pro_0']-train_reg['pro_0'].min())/(train_reg['pro_0'].max() - train_reg['pro_0'].min())
 train_reg['pro_100'] =(train_reg['pro_100']-train_reg['pro_100'].min())/(train_reg['pro_100'].max() - train_reg['pro_100'].min())
 train_reg['pro_200'] =(train_reg['pro_200']-train_reg['pro_200'].min())/(train_reg['pro_200'].max() - train_reg['pro_200'].min())
 train_reg['pro_300'] =(train_reg['pro_300']-train_reg['pro_300'].min())/(train_reg['pro_300'].max() - train_reg['pro_300'].min())
 train_reg['pro_400'] =(train_reg['pro_400']-train_reg['pro_400'].min())/(train_reg['pro_400'].max() - train_reg['pro_400'].min())
 train_reg['pro_500'] =(train_reg['pro_500']-train_reg['pro_500'].min())/(train_reg['pro_500'].max() - train_reg['pro_500'].min())
 train_reg['pro_600'] =(train_reg['pro_600']-train_reg['pro_600'].min())/(train_reg['pro_600'].max() - train_reg['pro_600'].min())
 train_reg['pro_700'] =(train_reg['pro_700']-train_reg['pro_700'].min())/(train_reg['pro_700'].max() - train_reg['pro_700'].min())
 train_reg['pro_800'] =(train_reg['pro_800']-train_reg['pro_800'].min())/(train_reg['pro_800'].max() - train_reg['pro_800'].min())
 train_reg['pro_900'] =(train_reg['pro_900']-train_reg['pro_900'].min())/(train_reg['pro_900'].max() - train_reg['pro_900'].min())
 train_reg['Sex'] =(train_reg['Sex'] - train_reg['Sex'].min())/(train_reg['Sex'].max() - train_reg['Sex'].min())
 train_reg['SmokingStatus'] =(train_reg['SmokingStatus'] - train_reg['SmokingStatus'].min())/(train_reg['SmokingStatus'].max() - train_reg['SmokingStatus'].min() )<init_hyperparams>
warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 warnings.simplefilter('ignore')
 sns.set()
%matplotlib inline
pd.set_option('display.max_columns', 500)
 filterwarnings('ignore')
 !pip install ycimpute
filterwarnings('ignore')
 pd.set_option('display.max_columns', None)
<prepare_x_and_y>
train=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )
FE = ['percent','SmokingStatus','Sex','age','BASE','volume','mean','skew','kurthosis','haa','median',
'midmean','midmedian','midskew','midkurthosis','midhaa',
'pro_0','pro_100','pro_200','pro_300','pro_400','pro_500',
'pro_600','pro_700','pro_800','pro_900']
train_reg['pre_s'] = -1
m,last3 = [],[]
for n in range(NFOLD):
tr_x,val_x = train_reg[train_reg.fold!=n][FE].values,train_reg[train_reg.fold==n][FE].values
tr_y,val_y = train_reg[train_reg.fold!=n]['slp'].values,train_reg[train_reg.fold==n]['slp'].values
randomforest = RandomForestRegressor(n_estimators = 500, oob_score = True, n_jobs = -1,
max_features = "sqrt", min_samples_leaf = 5, max_samples = 0.8)
 randomforest.fit(tr_x, tr_y)
 joblib.dump(randomforest, f'./all_rf_{n}.pkl')
 index = train_reg[train_reg.fold==n].index
train_reg.loc[index,['pre_s']] = randomforest.predict(val_x)
 s = np.mean(abs(train_reg.loc[index,['pre_s']].values-train_reg.loc[index,['slp']].values))
print(f"{n}th model's out-of-bag score : ", s)
 for p in train_reg[train_reg.fold==n].Patient:
percent_true = train_df.Percent.values[train_df.Patient == p]
fvc_true = train_df.FVC.values[train_df.Patient == p]
weeks_true = train_df.Weeks.values[train_df.Patient == p]
a = train_reg[train_reg.Patient==p].pre_s.values
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 train_df.loc[train_df.Patient==p,'img_fvc'] = fvc
train_df.loc[train_df.Patient==p,'img_con'] = percent
m.append(score_np(fvc_true, fvc,percent))
last3.append(score_np(fvc_true[-3:], fvc[-3:],percent[-3:]))
print(f'all_mean:{(np.mean(m)) },last3_mean:{np.mean(last3)}' )<load_from_csv>
def encode_FE(df1, df2, cols):
for col in cols:
df = pd.concat([df1[col],df2[col]])
 vc = df.value_counts(dropna=True, normalize=True ).to_dict()
vc[-1] = -1
nm = col+'_FE'
df1[nm] = df1[col].map(vc)
 df1[nm] = df1[nm].astype('float32')
 df2[nm] = df2[col].map(vc)
 df2[nm] = df2[nm].astype('float32')
 print(nm,', ',end='')
 def encode_LE(col,train,test,verbose=False):
df_comb = pd.concat([train[col],test[col]],axis=0)
 df_comb,_ = df_comb.factorize(sort=True)
 nm = col
if df_comb.max() >32000:
train[nm] = df_comb[:len(train)].astype('int32')
 test[nm] = df_comb[len(train):].astype('int32')
 else:
train[nm] = df_comb[:len(train)].astype('int16')
 test[nm] = df_comb[len(train):].astype('int16')
 del df_comb; x=gc.collect()
if verbose: print(nm,', ',end='')
 def encode_AG(main_columns, uids, aggregations, train_df, test_df,
fillna=True, usena=False):
for main_column in main_columns:
for col in uids:
for agg_type in aggregations:
new_col_name = main_column+'_'+col+'_'+agg_type
temp_df = pd.concat([train_df[[col, main_column]], test_df[[col,main_column]]])
 if usena: temp_df.loc[temp_df[main_column]==-1,main_column] = np.nan
temp_df = temp_df.groupby([col])[main_column].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train_df[new_col_name] = train_df[col].map(temp_df ).astype('float32')
 test_df[new_col_name] = test_df[col].map(temp_df ).astype('float32')
 if fillna:
train_df[new_col_name].fillna(-1,inplace=True)
 test_df[new_col_name].fillna(-1,inplace=True)
 print("'"+new_col_name+"'",', ',end='')
 def encode_CB(col1,col2,train,test):
nm = str(col1)+'_'+str(col2)
 train[nm] = train[col1].astype(str)+'_'+train[col2].astype(str)
 test[nm] = test[col1].astype(str)+'_'+test[col2].astype(str)
 print("'"+nm+"'",', ',end='')
 encode_LE(nm,train,test)
 def encode_AG2(main_columns, uids, train_df, test_df):
for main_column in main_columns:
for col in uids:
comb = pd.concat([train_df[[col]+[main_column]],test_df[[col]+[main_column]]],axis=0)
 mp = comb.groupby(col)[main_column].agg(['nunique'])['nunique'].to_dict()
train_df[col+'_'+main_column+'_ct'] = train_df[col].map(mp ).astype('float32')
 test_df[col+'_'+main_column+'_ct'] = test_df[col].map(mp ).astype('float32')
 print(col+'_'+main_column+'_ct, ',end='' )
test = pd.read_csv(f"{ROOT}/test.csv")
 test_df = test.merge(lung_stat_pd,)
 test_df.head()<feature_engineering>
main_columns=test._get_numeric_data().columns.drop(['Id'])
 categorical_columns=test.columns.drop(main_columns)
 categorical_columns=categorical_columns.drop(['Id'] )
train_reg = train_reg_bp
test_df['age'] =(test_df['Age'] - train_reg['Age'].min())/(train_reg['Age'].max() - train_reg['Age'].min())
 test_df['BASE'] =(test_df['FVC'] - train_reg['min_FVC'].min())/(train_reg['min_FVC'].max() - train_reg['min_FVC'].min())
 test_df['percent'] =(test_df['Percent'] - train_reg['min_Percent'].min())/(train_reg['min_Percent'].max() - train_reg['min_Percent'].min())
 test_df['volume'] =(test_df['Volume'] - train_reg['Volume'].min())/(train_reg['Volume'].max() - train_reg['Volume'].min())
 test_df['mean'] =(test_df['Mean'] - train_reg['Mean'].min())/(train_reg['Mean'].max() - train_reg['Mean'].min())
 test_df['skew'] =(test_df['Skew'] - train_reg['Skew'].min())/(train_reg['Skew'].max() - train_reg['Skew'].min())
 test_df['median'] =(test_df['Median'] - train_reg['Median'].min())/(train_reg['Median'].max() - train_reg['Median'].min())
 test_df['kurthosis'] =(test_df['Kurthosis'] - train_reg['Kurthosis'].min())/(train_reg['Kurthosis'].max() - train_reg['Kurthosis'].min())
 test_df['haa'] =(test_df['HAA'] - train_reg['HAA'].min())/(train_reg['HAA'].max() - train_reg['HAA'].min())
 test_df['midmedian'] =(test_df['midMedian'] - train_reg['midMedian'].min())/(train_reg['midMedian'].max() - train_reg['midMedian'].min())
 test_df['midmean'] =(test_df['midMean'] - train_reg['midMean'].min())/(train_reg['midMean'].max() - train_reg['midMean'].min())
 test_df['midskew'] =(test_df['midSkew'] - train_reg['midSkew'].min())/(train_reg['midSkew'].max() - train_reg['midSkew'].min())
 test_df['midkurthosis'] =(test_df['midKurthosis'] - train_reg['midKurthosis'].min())/(train_reg['midKurthosis'].max() - data['midKurthosis'].min())
 test_df['midhaa'] =(test_df['midHAA'] - train_reg['midHAA'].min())/(train_reg['midHAA'].max() - train_reg['midHAA'].min())
 test_df['pro_0'] =(test_df['pro_0']-train_reg['pro_0'].min())/(train_reg['pro_0'].max() - train_reg['pro_0'].min())
 test_df['pro_100'] =(test_df['pro_100']-train_reg['pro_100'].min())/(train_reg['pro_100'].max() - train_reg['pro_100'].min())
 test_df['pro_200'] =(test_df['pro_200']-train_reg['pro_200'].min())/(train_reg['pro_200'].max() - train_reg['pro_200'].min())
 test_df['pro_300'] =(test_df['pro_300']-train_reg['pro_300'].min())/(train_reg['pro_300'].max() - train_reg['pro_300'].min())
 test_df['pro_400'] =(test_df['pro_400']-train_reg['pro_400'].min())/(train_reg['pro_400'].max() - train_reg['pro_400'].min())
 test_df['pro_500'] =(test_df['pro_500']-train_reg['pro_500'].min())/(train_reg['pro_500'].max() - train_reg['pro_500'].min())
 test_df['pro_600'] =(test_df['pro_600']-train_reg['pro_600'].min())/(train_reg['pro_600'].max() - train_reg['pro_600'].min())
 test_df['pro_700'] =(test_df['pro_700']-train_reg['pro_700'].min())/(train_reg['pro_700'].max() - train_reg['pro_700'].min())
 test_df['pro_800'] =(test_df['pro_800']-train_reg['pro_800'].min())/(train_reg['pro_800'].max() - train_reg['pro_800'].min())
 test_df['pro_900'] =(test_df['pro_900']-train_reg['pro_900'].min())/(train_reg['pro_900'].max() - train_reg['pro_900'].min())
 test_df.loc[:,"Sex"] = pd.factorize(test_df.Sex)[0]
test_df.loc[:,"SmokingStatus"] = pd.factorize(test_df.SmokingStatus)[0]
test_df['Sex'] =(test_df['Sex'] - train_reg['Sex'].min())/(train_reg['Sex'].max() - train_reg['Sex'].min())
 test_df['SmokingStatus'] =(test_df['SmokingStatus'] - train_reg['SmokingStatus'].min())/(train_reg['SmokingStatus'].max() - train_reg['SmokingStatus'].min() )<merge>
encode_FE(train, test, [ 'GarageCars', 'ExterQual',"OverallQual","BsmtQual","Fireplaces","KitchenQual","CentralAir","FullBath","GrLivArea"])
 encode_CB('GarageCars', 'ExterQual',train,test)
 encode_CB("BsmtQual","OverallQual",train,test)
 encode_CB("Fireplaces","KitchenQual",train,test)
 encode_CB("CentralAir","FullBath",train,test)
 encode_CB('GarageCars_ExterQual','BsmtQual_OverallQual',train,test)
 encode_CB('Fireplaces_KitchenQual','CentralAir_FullBath' , train,test)
 encode_CB('GarageCars_ExterQual_BsmtQual_OverallQual' , 'Fireplaces_KitchenQual_CentralAir_FullBath' , train,test)
 encode_AG2(categorical_columns, ["MSSubClass","OverallQual"], train, test )
sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient")
 sub.head()
FE = ['percent','SmokingStatus','Sex','age','BASE','volume','mean','skew','kurthosis','haa','median',
'midmean','midmedian','midskew','midkurthosis','midhaa',
'pro_0','pro_100','pro_200','pro_300','pro_400','pro_500',
'pro_600','pro_700','pro_800','pro_900']
Y = ['slp']
pred_y = []
for n in range(NFOLD):
val_x = test_df[FE]
randomforest = joblib.load(f'./all_rf_{n}.pkl')
 pred_y.append(randomforest.predict(val_x))
pred_y = np.array(pred_y ).mean(0)
 test_df.loc[:,'preslp'] = pred_y
for p in test_df.Patient:
a = test_df[test_df.Patient==p].preslp.values
weeks_base = test_df[test_df.Patient==p].Weeks.values
weeks_true = sub[sub.Patient==p].Weeks.values
percent_true = sub[sub.Patient==p].Percent.values
fvc_true = sub[sub.Patient==p].FVC.values
fvc = a *(weeks_true - weeks_base[0])+ fvc_true[0]
percent = weeks_base[0] - a * abs(weeks_true - weeks_base[0])
 sub.loc[sub.Patient==p,'FVC'] = fvc
sub.loc[sub.Patient==p,'Confidence'] = percent
sub.head()
subs = sub[["Patient_Week","FVC","Confidence"]]<compute_test_metric>
encoder=OrdinalEncoder()
imputer=KNN()
def encode(data):

 nonulls = np.array(data.dropna())
 impute_reshape = nonulls.reshape(-1,1)
 impute_ordinal = encoder.fit_transform(impute_reshape)
 data.loc[data.notnull() ] = np.squeeze(impute_ordinal)
 return data
l = 0.
 m,last3 = [],[]
for i in train_df.Patient:
true_fvc = tr_1[tr_1.Patient==i].FVC
pred_fvc =(train_df[train_df.Patient==i].img_fvc)*l +(1-l)*tr_1[tr_1.Patient==i].pred_FVC
pred_sig =(train_df[train_df.Patient==i].img_con)*l +(1-l)*tr_1[tr_1.Patient==i].pred_Confidence
m.append(score_np(true_fvc, pred_fvc,pred_sig))
last3.append(score_np(true_fvc[-3:], pred_fvc[-3:],pred_sig[-3:]))
print(f'all_mean:{(np.mean(m)) },last3_mean:{np.mean(last3)}' )<train_model>
main_columns=test._get_numeric_data().columns.drop(['Id'])
 categorical_columns=test.columns.drop(main_columns)
 categorical_columns=categorical_columns.drop(['Id'] )
from fastai.train import Learner
from fastai.train import DataBunch
from fastai.callbacks import *
from fastai.basic_data import DatasetType<import_modules>
for i in categorical_columns:
lbe=preprocessing.LabelEncoder()
train[i]=lbe.fit_transform(train[i].astype(str))
test[i]=lbe.fit_transform(test[i].astype(str))
import numpy as np
import pandas as pd
import os
import time
import gc
import random
from tqdm._tqdm_notebook import tqdm_notebook as tqdm
from keras.preprocessing import text, sequence
import torch
from torch import nn
from torch.utils import data
from torch.nn import functional as F<set_options>
train=train.fillna(-1)
 test=test.fillna(-1 )
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<define_variables>
for i in categorical_columns:
if(test[i].max() == train[i].max())&(train[i].max() <10):
test = pd.get_dummies(test, columns = [i])
 train=pd.get_dummies(train, columns = [i] )
CRAWL_EMBEDDING_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
GLOVE_EMBEDDING_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220<feature_engineering>
y = train['SalePrice']
X= train.drop(['Id',"SalePrice"], axis=1)
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words<train_model>
models = [LGBMRegressor,
XGBRegressor,
GradientBoostingRegressor,
RandomForestRegressor,
DecisionTreeRegressor,
MLPRegressor,
KNeighborsRegressor,
SVR]
def sigmoid(x):
return 1 /(1 + np.exp(-x))
def train_model(learn,test,output_dim,lr=0.001,
batch_size=512, n_epochs=4,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(1)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds<normalization>
def compML(df, y, alg):
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.30,random_state=42)
 model = alg().fit(X_train, y_train)
 y_pred = model.predict(X_test)
 RMSE = np.sqrt(mean_squared_error(y_test, y_pred))
model_name = alg.__name__
print(model_name, "Model Test error:",RMSE )
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out<categorify>
for i in models:
compML(X, y, i )
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data<load_from_csv>
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.30,random_state=42)
 xgb = XGBRegressor().fit(X_train, y_train)
 y_pred = xgb.predict(X_test)
 np.sqrt(mean_squared_error(y_test, y_pred))
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )<prepare_x_and_y>
xgb_params = {"learning_rate": [0.1,0.01,0.5],
"max_depth": [2,3,4,5,8],
"n_estimators": [100,200,500,1000,3000,6000],
"colsample_bytree": [0.4,0.7,1],
"min_child_weight":[0,1,2],
"gamma":[0.6,0.8,0.2],
"subsample":[0.7,0.5,0.9],
"colsample_bytree":[0.7,0.01,0.2],
"nthread":[-1,-2],
"scale_pos_weight":[1,2,5,7],
"seed":[27,5,40,60],
"reg_alpha":[0.0000,0.005,0.5,0.00006]}
x_train = preprocess(train['comment_text'])
 y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = preprocess(test['comment_text'])
 identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()<data_type_conversions>
xgb_tuned = XGBRegressor(learning_rate=0.01,
n_estimators=6000,
max_depth=4,
min_child_weight=0,
gamma=0.6,
subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear',
nthread=-1,
scale_pos_weight=1,
seed=27,
reg_alpha=0.00006,
random_state=42 ).fit(X_train, y_train )
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T<define_variables>
y_pred = xgb_tuned.predict(X_test )
max_features = None<string_transform>
np.sqrt(mean_squared_error(y_test, y_pred))
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<define_variables>
from sklearn.metrics import mean_absolute_error
max_features = max_features or len(tokenizer.word_index)+ 1
max_features<choose_model_class>
print("Mean Absolute Error : " + str(mean_absolute_error(y_test, y_pred)) )
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))<choose_model_class>
ids=test["Id"]
test= test.drop(["Id"], axis=1 ).astype("float64" )
<concatenate><EOS> 
prediction=xgb_tuned.predict(test)
 output=pd.DataFrame({"Id":ids, "SalePrice":prediction})
 output.to_csv("submission_xgb.csv", index=False )