Kaggle competition: ASHRAE - Great Energy Predictor III
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<choose_model_class>
INPUT = ".. /input/ashrae-energy-prediction/"
df_train = pd.read_csv(f"{INPUT}train.csv")
 df_test = pd.read_csv(f"{INPUT}test.csv")
 bldg_metadata = pd.read_csv(f"{INPUT}building_metadata.csv")
 weather_train = pd.read_csv(f"{INPUT}weather_train.csv")
 weather_test = pd.read_csv(f"{INPUT}weather_test.csv")
 sample = pd.read_csv(f"{INPUT}sample_submission.csv" )
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )<compute_train_metric>
df_test = df_test.drop(columns=['row_id'] )
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<predict_on_test>
df_train = reduce_mem_usage(df=df_train)
 df_test = reduce_mem_usage(df=df_test)
 weather_train = reduce_mem_usage(df=weather_train)
 weather_test = reduce_mem_usage(df=weather_test )
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<compute_train_metric>
df_train = df_train.merge(bldg_metadata, on='building_id', how='left')
 df_test = df_test.merge(bldg_metadata, on='building_id', how='left')
 df_train = df_train.merge(weather_train, on=['site_id', 'timestamp'], how='left')
 df_test = df_test.merge(weather_test, on=['site_id', 'timestamp'], how='left' )
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
del weather_train, weather_test, bldg_metadata
gc.collect()
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<predict_on_test>
df_train['timestamp'] = pd.to_datetime(arg=df_train['timestamp'])
 df_test['timestamp'] = pd.to_datetime(arg=df_test['timestamp'] )
averaged_models.fit(train.values, y_train)
 stacked_train_pred = averaged_models.predict(train.values)
 print(rmsle(y_train, stacked_train_pred))<predict_on_test>
df_train['year'] = df_train['timestamp'].dt.year
df_train['month'] = df_train['timestamp'].dt.month
df_train['day'] = df_train['timestamp'].dt.day
df_train['hour'] = df_train['timestamp'].dt.hour
df_test['year'] = df_test['timestamp'].dt.year
df_test['month'] = df_test['timestamp'].dt.month
df_test['day'] = df_test['timestamp'].dt.day
df_test['hour'] = df_test['timestamp'].dt.hour
df_train['dayofweek'] = df_train['timestamp'].dt.dayofweek
df_test['dayofweek'] = df_test['timestamp'].dt.dayofweek
df_train['day-and-night'] = np.where(( df_train['hour'] < 6), 2, 1)
 df_test['day-and-night'] = np.where(( df_test['hour'] < 6), 2, 1)
 df_train.loc[df_train['month'].isin([12, 1, 2]), 'season'] = 1
df_train.loc[df_train['month'].isin([3, 4, 5]), 'season'] = 2
df_train.loc[df_train['month'].isin([6, 7, 8]), 'season'] = 3
df_train.loc[df_train['month'].isin([9, 10, 11]), 'season'] = 4
df_test.loc[df_test['month'].isin([12, 1, 2]), 'season'] = 1
df_test.loc[df_test['month'].isin([3, 4, 5]), 'season'] = 2
df_test.loc[df_test['month'].isin([6, 7, 8]), 'season'] = 3
df_test.loc[df_test['month'].isin([9, 10, 11]), 'season'] = 4
stacked_pred = averaged_models.predict(test)
 stacked_pred = np.expm1(stacked_pred )<predict_on_test>
df_train = reduce_mem_usage(df=df_train)
 df_test = reduce_mem_usage(df=df_test )
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))<predict_on_test>
df_train['age'] = df_train['year'] - df_train['year_built']
df_test['age'] = df_test['year'] - df_test['year_built']
new_df = df_train.groupby(by=['building_id'], as_index=False)['timestamp'].min()
new_df = new_df.rename(columns = {'timestamp': 'start_ts'})
 df_train = df_train.merge(new_df, on = 'building_id', how='left')
 df_test = df_test.merge(new_df, on = 'building_id', how='left')
 df_train['hours_passed'] =(df_train['timestamp'] - df_train['start_ts'] ).dt.total_seconds() /3600
df_test['hours_passed'] =(df_test['timestamp'] - df_test['start_ts'] ).dt.total_seconds() /3600
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))<compute_test_metric>
df_train = reduce_mem_usage(df=df_train)
 df_test = reduce_mem_usage(df=df_test )
print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))<define_variables>
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15<save_to_csv>
df_train = df_train.query('not(site_id==0 & timestamp<"2016-05-21 00:00:00")')
 df_train = df_train.loc[df_train['meter_reading'] > 0, :]
cols = ['floor_count', 'air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed']
df_train.loc[:, cols] = df_train.loc[:, cols].interpolate(axis=0)
 df_test.loc[:, cols] = df_test.loc[:, cols].interpolate(axis=0 )
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False )<set_options>
cat_cols = ['meter', 'primary_use', 'site_id', 'building_id', 'year', 'month', 'day', 'hour', 'dayofweek', 'season', 'day-and-night']
for col in cat_cols:
df_train[col] = df_train[col].astype('category')
 df_test[col] = df_test[col].astype('category' )
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<define_variables>
df_train = df_train.reset_index(drop=True )
CRAWL_EMBEDDING_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
GLOVE_EMBEDDING_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
NUMBERBATCH_PATH = '.. /input/conceptnet-numberbatch-vectors/numberbatch-en-17.06.txt/numberbatch-en-17.06.txt'
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220
EPOCHS = 4
BATCH_SIZE = 512
MAX_LEN = 220<feature_engineering>
df_val = df_train.loc[df_train['timestamp'] >= '2016-11-01 00:00:00']
df_train = df_train.loc[df_train['timestamp'] < '2016-11-01 00:00:00']
y_train = df_train['meter_reading']
y_val = df_val['meter_reading']
y_train = np.log1p(y_train)
 y_val = np.log1p(y_val)
 df_train = df_train.drop(columns=['meter_reading'])
 df_val = df_val.drop(columns=['meter_reading'])
 df_train = df_train.drop(columns=['timestamp', 'start_ts'])
 df_val = df_val.drop(columns=['timestamp', 'start_ts'])
 df_test = df_test.drop(columns=['timestamp', 'start_ts'] )
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words<train_model>
gc.collect()
def sigmoid(x):
return 1 /(1 + np.exp(-x))
def train_model(model, train, test, loss_fn, output_dim, lr=0.001,
batch_size=512, n_epochs=4,
enable_checkpoint_ensemble=True):
param_lrs = [{'params': param, 'lr': lr} for param in model.parameters() ]
optimizer = torch.optim.Adam(param_lrs, lr=lr)
 scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda epoch: 0.6 ** epoch)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
for epoch in range(n_epochs):
start_time = time.time()
scheduler.step()
model.train()
avg_loss = 0.
 for data in tqdm(train_loader, disable=False):
x_batch = data[:-1]
y_batch = data[-1]
y_pred = model(*x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
y_pred = sigmoid(model(*x_batch ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, elapsed_time))
if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds<normalization>
lgbmr = LGBMRegressor(n_estimators=500, random_state=10)
 lgbmr.fit(df_train, y_train)
 y_predict = lgbmr.predict(df_val)
 score = np.sqrt(mse(y_val, y_predict))
print(f"score: {score}")
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out<categorify>
lgbmr_errors.sort_values(by='error', ascending=False ).head()
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data<prepare_x_and_y>
X_1st_half = df_train[:int(df_train.shape[0]/2)]
y_1st_half = y_train[:int(df_train.shape[0]/2)]
X_2nd_half = df_train[int(df_train.shape[0]/2):]
y_2nd_half = y_train[int(df_train.shape[0]/2):]
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_train = preprocess(train['comment_text'])
 y_train = np.where(train['target'] >= 0.5, 1, 0)
 y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = preprocess(test['comment_text'])
<string_transform>
lgbmr_1st_half = LGBMRegressor(random_state=10)
 lgbmr_2nd_half = LGBMRegressor(random_state=10)
 lgbmr_1st_half.fit(X_1st_half, y_1st_half)
 lgbmr_2nd_half.fit(X_2nd_half, y_2nd_half)
 y_predict_1 = lgbmr_1st_half.predict(df_val)
 y_predict_2 = lgbmr_2nd_half.predict(df_val)
 y_predict_1_2 =(pd.Series(data=y_predict_1, name='prediction_1')+ pd.Series(data=y_predict_2, name='prediction_2')) /2
score = np.sqrt(mse(y_val, y_predict_1_2))
print(f"score: {score}" )
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<define_variables>
X_1st_half = df_train[:int(pd.concat([df_train, df_val], axis=0 ).shape[0]/2)]
y_1st_half = y_train[:int(pd.concat([df_train, df_val], axis=0 ).shape[0]/2)]
X_2nd_half = df_train[int(pd.concat([df_train, df_val], axis=0 ).shape[0]/2):]
y_2nd_half = y_train[int(pd.concat([df_train, df_val], axis=0 ).shape[0]/2):]
lgbmr_1st_half = LGBMRegressor(random_state=10)
 lgbmr_2nd_half = LGBMRegressor(random_state=10)
 lgbmr_1st_half.fit(X_1st_half, y_1st_half)
 lgbmr_2nd_half.fit(X_2nd_half, y_2nd_half )
max_features = None
max_features = max_features or len(tokenizer.word_index)+ 1
max_features<choose_model_class>
filename = 'lgbm_model1.pickle'
pickle.dump(lgbmr, open(filename, 'wb'))
loaded_model = pickle.load(open(filename, 'rb'))
filename = 'lgbmr_1st_half.pickle'
pickle.dump(lgbmr, open(filename, 'wb'))
lgbmr_1st_half = pickle.load(open(filename, 'rb'))
filename = 'lgbmr_2nd_half.pickle'
pickle.dump(lgbmr, open(filename, 'wb'))
lgbmr_2nd_half = pickle.load(open(filename, 'rb'))
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))
print('n words(crawl): ', len(crawl_matrix))<choose_model_class>
del df_train, df_val, y_train, y_val, lgbmr
gc.collect()
glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))
print('n words(crawl): ', len(glove_matrix))<define_variables>
STEP = 1000000
y_test_predict = []
for i in range(0, df_test.shape[0], STEP):
batch_prediction = lgbmr_1st_half.predict(df_test.loc[i:i+STEP-1,:])
 y_test_predict.append(list(batch_prediction))
y_test_1st_half = []
for predictions in y_test_predict:
y_test_1st_half = y_test_1st_half + predictions
STEP = 1000000
y_test_predict = []
for i in range(0, df_test.shape[0], STEP):
batch_prediction = lgbmr_2nd_half.predict(df_test.loc[i:i+STEP-1,:])
 y_test_predict.append(list(batch_prediction))
y_test_2nd_half = []
for predictions in y_test_predict:
y_test_2nd_half = y_test_2nd_half + predictions
numb_matrix, unknown_words_numberbatch = build_matrix(tokenizer.word_index, NUMBERBATCH_PATH)
 print('n unknown words(numberbatch): ', len(unknown_words_glove))
print('n words(numberbatch): ', len(numb_matrix))<data_type_conversions>
sample['meter_reading'] =(pd.Series(data=y_test_1st_half, name='pred_1st_half')+
pd.Series(data=y_test_2nd_half, name='pred_2nd_half')) /2
x_train_torch = torch.tensor(x_train, dtype=torch.long ).cuda()
x_test_torch = torch.tensor(x_test, dtype=torch.long ).cuda()
y_train_torch = torch.tensor(np.hstack([y_train[:, np.newaxis], y_aux_train]), dtype=torch.float32 ).cuda()<train_model>
sample['meter_reading'] = np.expm1(sample['meter_reading'] )
<save_to_csv><EOS> 
sample.to_csv("submission.csv", index=False, float_format='%.4f' )