Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<categorify>
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
import numpy as np
import pandas as pd
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sklearn
import seaborn as sns
from sklearn.preprocessing import Imputer
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
encoded_train = tokenizer.batch_encode_plus(
 df_train[df_train.set_type=='train'].text.values,
add_special_tokens=True,
return_attention_masks=True,
pad_to_max_length=True,
max_length=256,
return_tensors='pt' 
)
 encoded_val = tokenizer.batch_encode_plus(
 df_train[df_train.set_type=='val'].text.values,
add_special_tokens=True,
return_attention_masks=True,
pad_to_max_length=True,
max_length=256,
return_tensors='pt' 
)
 encoded_test = tokenizer.batch_encode_plus(
 df_test.text.values,
add_special_tokens=True,
return_attention_masks=True,
pad_to_max_length=True,
max_length=256,
return_tensors='pt' 
)
 input_ids_train = encoded_train['input_ids']
attention_masks_train = encoded_train['attention_mask']
labels_train = torch.tensor(df_train[df_train.set_type=='train'].target.values)
 input_ids_val = encoded_val['input_ids']
attention_masks_val = encoded_val['attention_mask']
labels_val = torch.tensor(df_train[df_train.set_type=='val'].target.values)
 input_ids_test = encoded_test['input_ids']
attention_masks_test = encoded_test['attention_mask']<create_dataframe>
df_train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 df_train.head()
df_test=pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 df_test.shape
dataset_train = TensorDataset(input_ids_train, attention_masks_train, labels_train)
 dataset_val = TensorDataset(input_ids_val, attention_masks_val, labels_val)
 dataset_test = TensorDataset(input_ids_test, attention_masks_test )<load_pretrained>
train_ID= df_train['Id']
train_ID
test_ID= df_test['Id']
test_ID
model = BertForSequenceClassification.from_pretrained('bert-base-uncased',
num_labels = 2,
output_attentions = False,
output_hidden_states = False
)<load_pretrained>
df_test.drop('Id',axis=1,inplace=True)
 df_test.drop('Utilities',axis=1,inplace=True)
 df_test.shape
y_train=df_train.SalePrice.values
y_train
dataloader_train = DataLoader(
 dataset_train,
sampler= RandomSampler(dataset_train),
batch_size=32 
)
 dataloader_val = DataLoader(
 dataset_val,
sampler = SequentialSampler(dataset_val),
batch_size=32 
)
 dataloader_test = DataLoader(
 dataset_test,
sampler = SequentialSampler(dataset_test),
batch_size=32
)<choose_model_class>
df_train.SalePrice = np.log1p(df_train.SalePrice )
optimizer = AdamW(model.parameters() ,
lr=1e-5,
eps=1e-8)
 epochs = 4
scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=len(dataloader_train)*epochs
)<compute_test_metric>
y_train = df_train.SalePrice.values
def f1_score_func(preds, labels):
preds_flat = np.argmax(preds, axis =1 ).flatten()
labels_flat = labels.flatten()
return f1_score(labels_flat, preds_flat, average='weighted')
 def accuracy_per_class(preds, labels):
preds_flat = np.argmax(preds, axis =1 ).flatten()
labels_flat = labels.flatten()
for label in np.unique(labels_flat):
y_preds = preds_flat[labels_flat==label]
y_true = labels_flat[labels_flat==label]
print(f'Class: {label}')
 print(f'Accuracy: {len(y_preds[y_preds==label])}/{len(y_true)}' )<set_options>
train_miss = df_train.isnull().sum()
print(train_miss)
 train_miss.sort_values(ascending=False)
 print(train_miss )
seed_val = 10
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val)
 device = 'cuda' if torch.cuda.is_available() else 'cpu'
model.to(device)
 print(device)
<predict_on_test>
test_miss = df_test.isnull().sum()
print(test_miss)
 test_miss.sort_values(ascending=False)
 print(test_miss )
def evaluate(dataloader_val):
model.eval()
loss_val_total = 0
predictions, true_vals = [], []
for batch in dataloader_val:
batch = tuple(b.to(device)for b in batch)
 inputs = {'input_ids': batch[0],
'attention_mask': batch[1],
'labels': batch[2],
}
with torch.no_grad() :
outputs = model(**inputs)
 loss = outputs[0]
logits = outputs[1]
loss_val_total += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = inputs['labels'].cpu().numpy()
predictions.append(logits)
 true_vals.append(label_ids)
 loss_val_avg = loss_val_total/len(dataloader_val)
 predictions = np.concatenate(predictions, axis=0)
 true_vals = np.concatenate(true_vals, axis=0)
 return loss_val_avg, predictions, true_vals 
<train_model>
commonNa = [
'MSZoning','Electrical','KitchenQual',
'Exterior1st','Exterior2nd','SaleType',
'LotFrontage','Functional'
]
toZero = [
'MasVnrArea','GarageYrBlt','BsmtHalfBath',
'BsmtFullBath','GarageArea','GarageCars',
'BsmtFinSF1','BsmtFinSF2','BsmtUnfSF',
'TotalBsmtSF'
]
toNoData = [
'PoolQC','MiscFeature','Alley',
'Fence','FireplaceQu','GarageType',
'GarageFinish','GarageQual',
'GarageCond','BsmtQual','BsmtCond',
'BsmtExposure','BsmtFinType1','BsmtFinType2',
'MasVnrType'
]
for epoch in tqdm(range(1, epochs+1)) :
model.train()
training_loss=0
progress_bar = tqdm(dataloader_train, desc='Epoch {:1d}'.format(epoch),
leave=False,
disable=False 
)
 for batch in progress_bar:
model.zero_grad()
batch = tuple(b.to(device)for b in batch)
 inputs = {
'input_ids': batch[0],
'attention_mask':batch[1],
'labels':batch[2]
}
outputs = model(**inputs)
 loss=outputs[0]
training_loss+= loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
progress_bar.set_postfix({'training_loss': '{:.3f}'.format(loss.item() /len(batch)) })
 tqdm.write(f'
Epoch {epoch}')
 loss_train_avg = training_loss/len(dataloader_train)
 tqdm.write(f'Training loss: {loss_train_avg}')
 val_loss, predictions, true_vals = evaluate(dataloader_val)
 val_f1 = f1_score_func(predictions, true_vals)
 tqdm.write(f'Validation Loss: {val_loss}')
 tqdm.write(f'F1 score: {val_f1}')
<prepare_output>
def fillNan(df):
df['Functional']=df['Functional'].fillna('Typ')
 for i in commonNa:
df[i]=df[i].fillna(df[i].mode() [0])
 for i in toNoData:
df[i]=df[i].fillna('None')
 for i in toZero:
df[i]=df[i].fillna(0)
 return df
_, predictions, true_vals = evaluate(dataloader_val )<compute_test_metric>
df_train =fillNan(df_train)
 df_train.isnull().sum()
accuracy_per_class(predictions, true_vals )<predict_on_test>
df_test =fillNan(df_test)
 df_test.isnull().sum()
model.eval()
predictions=[]
for batch in dataloader_test:
batch = tuple(b.to(device)for b in batch)
 inputs = {'input_ids': batch[0],
'attention_mask': batch[1]
}
with torch.no_grad() :
outputs = model(**inputs)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
predictions.append(np.argmax(logits,axis=1)) 
<prepare_output>
def skew_(df):
colls = [col for col in df.columns if df[col].dtype in ['int64','float']]
skews_df = [col for col in df[colls].columns if df[col].skew() >.7]
def skewfix(data, data2):
for i in data2:
data[i] = np.log1p(data[i])
 return data
return skewfix(df, skews_df)
 df_train, df_test = skew_(df_train), skew_(df_test)
prediction = list(chain.from_iterable(predictions)) 
<load_from_csv>
encoder = LabelEncoder()
def encode(df):
cat_df = [col for col in df.columns if df[col].dtype not in ['int','float']]
for col in cat_df:
df[col] = encoder.fit_transform(df[col])
 return df
df_test,df_train=encode(df_test),encode(df_train )
sub= pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 sub.head()<prepare_output>
X=df_train.drop(['Id', 'SalePrice'], axis=1)
 y=df_train.SalePrice.values
sub['target']=prediction<save_to_csv>
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.35,random_state=100 )
sub.to_csv('submission.csv', index=False )<save_to_csv>
regressor = SVR()
regressor.fit(X_train,y_train)
 svrpredict=regressor.predict(X_test )
sub.to_csv('submission.csv', index=False )<import_modules>
for i in range(0,433):
print("Error in value number",i,(y_test[i]-svrpredict[i]))
Bidirectional, Lambda, Conv1D, MaxPooling1D, GRU,GlobalMaxPooling1D,GlobalAveragePooling1D, concatenate 
<set_options>
mean_squared_error(y_test,svrpredict )
class CyclicLR(Callback):
def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr() )<load_from_csv>
model = xgboost.XGBRegressor(colsample_bytree=0.4,
gamma=0,
learning_rate=0.07,
max_depth=3,
min_child_weight=1.5,
n_estimators=10000,
reg_alpha=0.75,
reg_lambda=0.45,
subsample=0.6,
seed=42)
 model.fit(X_train,y_train)
 xgb_prediction = model.predict(X_test)
 def inv_y(transformed_y):
return np.exp(transformed_y)
data = pd.read_csv(".. /input/nlp-getting-started/train.csv" )<string_transform>
for i in range(0,433):
print("Error in value number",i,(y_test[i]-xgb_prediction[i]))
MAX_SEQUENCE_LENGTH = 60
MAX_NB_WORDS = 30000
EMBEDDING_DIM = 300
tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
 tokenizer.fit_on_texts(data['text'].values)
 sequences = tokenizer.texts_to_sequences(data['text'].values)
 word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
pad_text = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH )<feature_engineering>
mean_squared_error(y_test,xgb_prediction )
embeddings_index = {}
f = open('.. /input/glove840b300dtxt/glove.840B.300d.txt','r',encoding='utf-8')
 for line in f:
values = line.split(' ')
 word = values[0]
coefs = np.asarray([float(val)for val in values[1:]])
 embeddings_index[word] = coefs
f.close()
print('
Found %s word vectors.' % len(embeddings_index))
embedding_matrix = np.zeros(( len(word_index)+ 1, EMBEDDING_DIM))
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
print('Found %s word vectors.' % len(embedding_matrix))
del embedding_vector,embeddings_index<split>
X=df_train.drop(['Id', 'SalePrice'], axis=1)
 X4=X.iloc[:-1,:]
X5=X4.drop('Utilities',axis=1)
 y=df_train.iloc[:-1,:].SalePrice.values
X5.shape
df_test.shape
y.shape
model = xgboost.XGBRegressor(colsample_bytree=0.4,
gamma=0,
learning_rate=0.07,
max_depth=3,
min_child_weight=1.5,
n_estimators=10000,
reg_alpha=0.75,
reg_lambda=0.45,
subsample=0.6,
seed=42)
 model.fit(X5,y)
 xgb_prediction = model.predict(df_test)
 def inv_y(transformed_y):
return np.exp(transformed_y)
<choose_model_class><EOS> 
submission=pd.DataFrame({ "Id": test_ID, "SalePrice": inv_y(xgb_prediction)})
 submission.to_csv('Submissionfile.csv',index=False )