Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<data_type_conversions>
%matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
%%time
test_df['comment_text'] = preprocess(test_df['comment_text'] ).astype(str )<drop_column>
print(check_output(["ls", ".. /input"] ).decode("utf8"))
df_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
test = test_df<choose_model_class>
df_train = df_train.drop(df_train[(df_train['GrLivArea']>4000)&(df_train['SalePrice']<300000)].index)
 ntrain = df_train.shape[0]
ntest = df_test.shape[0]
y_train = df_train.SalePrice.values
all_data = pd.concat(( df_train, df_test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))
class BertTokenizeProcessor(TokenizeProcessor):
def __init__(self, tokenizer):
super().__init__(tokenizer=tokenizer, include_bos=False, include_eos=False)
 class BertNumericalizeProcessor(NumericalizeProcessor):
def __init__(self, *args, **kwargs):
super().__init__(*args, vocab=Vocab(list(bert_tokenizer.vocab.keys())) , **kwargs)
 def get_bert_processor(tokenizer:Tokenizer=None, vocab:Vocab=None):

 return [BertTokenizeProcessor(tokenizer=tokenizer),
NumericalizeProcessor(vocab=vocab)]<create_dataframe>
NA=all_data[['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond','GarageYrBlt','BsmtFinType2','BsmtFinType1','BsmtCond', 'BsmtQual','BsmtExposure', 'MasVnrArea','MasVnrType','Electrical','MSZoning','BsmtFullBath','BsmtHalfBath','Utilities','Functional','Exterior1st','BsmtUnfSF','Exterior2nd','TotalBsmtSF','GarageArea','GarageCars','KitchenQual','BsmtFinSF2','BsmtFinSF1','SaleType']]
class BertDataBunch(TextDataBunch):
@classmethod
def from_df(cls, path:PathOrStr, train_df:DataFrame, valid_df:DataFrame, test_df:Optional[DataFrame]=None,
tokenizer:Tokenizer=None, vocab:Vocab=None, classes:Collection[str]=None, text_cols:IntsOrStrs=1,
label_cols:IntsOrStrs=0, label_delim:str=None, **kwargs)-> DataBunch:
"Create a `TextDataBunch` from DataFrames."
p_kwargs, kwargs = split_kwargs_by_func(kwargs, get_bert_processor)
 processor = get_bert_processor(tokenizer=tokenizer, vocab=vocab, **p_kwargs)
 if classes is None and is_listy(label_cols)and len(label_cols)> 1: classes = label_cols
src = ItemLists(path, TextList.from_df(train_df, path, cols=text_cols, processor=processor),
TextList.from_df(valid_df, path, cols=text_cols, processor=processor))
src = src.label_for_lm() if cls==TextLMDataBunch else src.label_from_df(cols=label_cols, classes=classes)
 if test_df is not None: src.add_test(TextList.from_df(test_df, path, cols=text_cols))
return src.databunch(**kwargs )<choose_model_class>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head(20 )
bert_model = BertForSequenceClassification.from_pretrained(BERT_MODEL_BIN, num_labels=2)
 loss_func = nn.CrossEntropyLoss()<load_pretrained>
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None' )
learn = load_learner('.. /input/fastai-sigsaw-1-epoch', 'train_epoch_9.pkl' )<create_dataframe>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )
test_list = TextList.from_df(test, cols=['comment_text'], path='.. /input/jigsaw-unintended-bias-in-toxicity-classification' )<feature_engineering>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0 )
learn.data.add_test(test_list )<predict_on_test>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None')
 all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0)
 all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0])
 all_data = all_data.drop(['Utilities'], axis=1)
 all_data["Functional"] = all_data["Functional"].fillna("Typ")
 all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0])
 all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0])
 all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0])
 all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0])
 all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None")
preds = learn.get_preds(DatasetType.Test )<predict_on_test>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head(20 )
preds = [learn.predict(test['comment_text'][i])for i in tqdm(range(len(test_df)))]<concatenate>
total = df_train.isnull().sum().sort_values(ascending=False)
 percent =(df_train.isnull().sum() /df_train.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(20 )
preds_sub = [np.array(x[2][1])for x in tqdm(preds)]<prepare_output>
cols =('BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))
submission_bert = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': preds_sub
})
 submission_bert.head()<save_to_csv>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
submission_bert.to_csv("submission.csv", index=False )<define_variables>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.5
for feature in skewed_features:
all_data[feature] = boxcox1p(all_data[feature], lam )
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220<load_from_csv>
all_data = pd.get_dummies(all_data)
 print(all_data.shape )
train_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 sample_df = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv' )<define_variables>
df_train = all_data[:ntrain]
df_test = all_data[ntrain:]
IDENTITY_COLUMNS = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness'
]
AUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'<prepare_x_and_y>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(df_train.values)
 rmse= np.sqrt(-cross_val_score(model, df_train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
x_train = train_df[TEXT_COLUMN].astype(str)
 y_train = train_df[TARGET_COLUMN].values
y_aux_train = train_df[AUX_COLUMNS].values
x_test = test_df[TEXT_COLUMN].astype(str)
 for column in IDENTITY_COLUMNS + [TARGET_COLUMN]:
train_df[column] = np.where(train_df[column] >= 0.5, True, False )<feature_engineering>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
tokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE)
 tokenizer.fit_on_texts(list(x_train)+ list(x_test))<string_transform>
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test )<concatenate>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN)
 print(x_train[0] )<define_variables>
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
sample_weights = np.ones(len(x_train), dtype=np.float32 )<define_variables>
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
sample_weights += train_df[IDENTITY_COLUMNS].sum(axis=1)
<define_variables>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.005,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=5,
loss='huber', random_state =5)
 score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
sample_weights += train_df[TARGET_COLUMN] *(~train_df[IDENTITY_COLUMNS] ).sum(axis=1)
 sample_weights +=(~train_df[TARGET_COLUMN])* train_df[IDENTITY_COLUMNS].sum(axis=1)* 5
sample_weights /= sample_weights.mean()<categorify>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in f)
 def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
pass
return embedding_matrix<define_variables>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]
embedding_file1 = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
embedding_file2 = '.. /input/glove840b300dtxt/glove.840B.300d.txt'<concatenate>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1)
embedding_matrix = np.concatenate(
 [build_matrix(tokenizer.word_index, f)for f in EMBEDDING_FILES], axis=-1 )<define_variables>
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
EPOCHS = 4<choose_model_class>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
def build_model(embedding_matrix, num_aux_targets):
words = Input(shape=(None,))
x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False )(words)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)
 hidden = concatenate([
GlobalMaxPooling1D()(x),
GlobalAveragePooling1D()(x),
])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu' )(hidden)])
 result = Dense(1, activation='sigmoid' )(hidden)
 aux_result = Dense(num_aux_targets, activation='sigmoid' )(hidden)
 model = Model(inputs=words, outputs=[result, aux_result])
 model.compile(loss='binary_crossentropy', optimizer='adam')
 return model<choose_model_class>
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
model = build_model(embedding_matrix, y_aux_train.shape[-1] )<define_variables>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
package_dir = ".. /input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT"
sys.path.append(package_dir )<set_options>
stacked_averaged_models.fit(df_train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(df_train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(df_test.values))
print(rmsle(y_train, stacked_train_pred))
%reload_ext autoreload
%autoreload 2
%matplotlib inline 
<string_transform>
model_xgb.fit(df_train, y_train)
 xgb_train_pred = model_xgb.predict(df_train)
 xgb_pred = np.expm1(model_xgb.predict(df_test))
print(rmsle(y_train, xgb_train_pred))
def convert_lines(example, max_seq_length,tokenizer):
max_seq_length -=2
all_tokens = []
longer = 0
for text in tqdm(example):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)>max_seq_length:
tokens_a = tokens_a[:max_seq_length]
longer += 1
one_token = tokenizer.convert_tokens_to_ids(["[CLS]"]+tokens_a+["[SEP]"])+[0] *(max_seq_length - len(tokens_a))
all_tokens.append(one_token)
 return np.array(all_tokens)
 def is_interactive() :
return 'SHLVL' not in os.environ
def seed_everything(seed=123):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
emb_arr = KeyedVectors.load(path)
 return emb_arr
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( max_features + 1, 300))
unknown_words = []
for word, i in word_index.items() :
if i <= max_features:
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.lower() ]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.title() ]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
def sigmoid(x):
return 1 /(1 + np.exp(-x))
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
def train_model(learn,test,output_dim,lr=0.001,
batch_size=512, n_epochs=3,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(1)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds
def handle_punctuation(x):
x = x.translate(remove_dict)
 x = x.translate(isolate_dict)
 return x
def handle_contractions(x):
x = tokenizer.tokenize(x)
 return x
def fix_quote(x):
x = [x_[1:] if x_.startswith("'")else x_ for x_ in x]
x = ' '.join(x)
 return x
def preprocess(x):
x = handle_punctuation(x)
 x = handle_contractions(x)
 x = fix_quote(x)
 return x
class SequenceBucketCollator() :
def __init__(self, choose_length, sequence_index, length_index, label_index=None):
self.choose_length = choose_length
self.sequence_index = sequence_index
self.length_index = length_index
self.label_index = label_index
def __call__(self, batch):
batch = [torch.stack(x)for x in list(zip(*batch)) ]
sequences = batch[self.sequence_index]
lengths = batch[self.length_index]
length = self.choose_length(lengths)
 mask = torch.arange(start=maxlen, end=0, step=-1)< length
padded_sequences = sequences[:, mask]
batch[self.sequence_index] = padded_sequences
if self.label_index is not None:
return [x for i, x in enumerate(batch)if i != self.label_index], batch[self.label_index]
return batch
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x, lengths=None):
h_embedding = self.embedding(x.long())
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out
def custom_loss(data, targets):
bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ bce_loss_2
def reduce_mem_usage(df):
start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
def ensemble_predictions(predictions, weights, type_="linear"):
assert np.isclose(np.sum(weights), 1.0)
 if type_ == "linear":
res = np.average(predictions, weights=weights, axis=0)
 elif type_ == "harmonic":
res = np.average([1 / p for p in predictions], weights=weights, axis=0)
 return 1 / res
elif type_ == "geometric":
numerator = np.average(
 [np.log(p)for p in predictions], weights=weights, axis=0 
)
 res = np.exp(numerator / sum(weights))
return res
elif type_ == "rank":
res = np.average([rankdata(p)for p in predictions], weights=weights, axis=0)
 return res /(len(res)+ 1)
 return res<define_variables>
model_lgb.fit(df_train, y_train)
 lgb_train_pred = model_lgb.predict(df_train)
 lgb_pred = np.expm1(model_lgb.predict(df_test.values))
print(rmsle(y_train, lgb_train_pred))
warnings.filterwarnings(action='once')
 device = torch.device('cuda')
 MAX_SEQUENCE_LENGTH = 300
SEED = 1234
BATCH_SIZE = 512
BERT_MODEL_PATH = '.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
bert_config = BertConfig('.. /input/bert-inference/bert/bert_config.json')
 tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None,do_lower_case=True)
 tqdm.pandas()
CRAWL_EMBEDDING_PATH = '.. /input/gensim-embeddings-dataset/crawl-300d-2M.gensim'
GLOVE_EMBEDDING_PATH = '.. /input/gensim-embeddings-dataset/glove.840B.300d.gensim'
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220
if not is_interactive() :
def nop(it, *a, **k):
return it
tqdm = nop
fastprogress.fastprogress.NO_BAR = True
master_bar, progress_bar = force_console_behavior()
fastai.basic_train.master_bar, fastai.basic_train.progress_bar = master_bar, progress_bar
seed_everything()<load_from_csv>
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15
<choose_model_class><EOS> 
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False)
 print("Submitted Successfully...!" )