Kaggle competition: ASHRAE - Great Energy Predictor III
sum(sample_submission['segment_id']==test_set['segment_id'])/ len(test_set )<feature_engineering>
%matplotlib inline
pd.set_option('display.max_columns',100)
 pd.set_option('display.max_rows',1500)
 pd.set_option('display.float_format', lambda x: '%.5f' % x)
 init_notebook_mode(connected=True)
train_set['time_to_eruption_pc'] = train_set['time_to_eruption'] / train_set['time_to_eruption'].max()
NFOLDS=10
skf = StratifiedKFold(n_splits=NFOLDS)
 train_set['label_strat'] = np.round(train_set['time_to_eruption_pc'] * 20, 0)
 train_set['fold']=0
f=0
for trn_idx, val_idx in skf.split(train_set[['segment_id']], train_set['label_strat']):
train_set.loc[val_idx, 'fold']=f
f+=1
train_set['fold'].value_counts()<define_variables>
metadata_dtype = {'site_id':"uint8",'building_id':'uint16','square_feet':'float32','year_built':'float32','floor_count':"float16"}
metadata = pd.read_csv(".. /input/ashrae-energy-prediction/building_metadata.csv",dtype=metadata_dtype)
 metadata.info(memory_usage='deep' )
DROP_FTS = ['segment_id', 'time_to_eruption','h:m:s', 'label_strat', 'fold', 'time_to_eruption_pc',]
SEL_FTS = [x for x in train_set.columns if x not in DROP_FTS]
LABEL = 'time_to_eruption'<train_model>
weather_dtype = {"site_id":"uint8"}
weather_train = pd.read_csv(".. /input/ashrae-energy-prediction/weather_train.csv",parse_dates=['timestamp'],dtype=weather_dtype)
 weather_test = pd.read_csv(".. /input/ashrae-energy-prediction/weather_test.csv",parse_dates=['timestamp'],dtype=weather_dtype)
 print(weather_train.info(memory_usage='deep'))
print("-------------------------------------")
 print(weather_test.info(memory_usage='deep'))
optuna.logging.set_verbosity(optuna.logging.WARNING)
 OPTUNA_TRIALS = 30
class Optimizer:
def __init__(self, metric, trials=OPTUNA_TRIALS):
self.metric = metric
self.trials = trials
self.sampler = TPESampler(seed=42)
 def objective(self, trial):
model = create_model(trial)
 model.fit(X_train, y_train)
 preds = model.predict(X_val)
 if self.metric == 'mae':
return -mae(y_val, preds)
 else:
return -np.sqrt(mse(y_val, preds))
def optimize(self):
study = optuna.create_study(direction="maximize", sampler=self.sampler)
 study.optimize(self.objective, n_trials=self.trials)
 return study.best_params
def create_model(trial):
colsample_bytree = trial.suggest_uniform("colsample_bytree", 0.2, 0.95)
 subsample = trial.suggest_uniform("subsample", 0.2, 0.9)
 reg_alpha = trial.suggest_uniform("reg_alpha", 0.001, 100)
 max_depth = trial.suggest_int("max_depth", 1, 15)
 min_child_samples = trial.suggest_int('min_child_samples', 1, 15)
 n_estimators = trial.suggest_int('n_estimators', 100, 250)
 model = LGBMRegressor(random_state = 42,
max_depth = max_depth,
n_estimators = n_estimators,
colsample_bytree=colsample_bytree,
subsample=subsample,
reg_alpha=reg_alpha,
learning_rate = 0.05)
 return model<prepare_x_and_y>
train_dtype = {'meter':"uint8",'building_id':'uint16','meter_reading':"float32"}
train = pd.read_csv(".. /input/ashrae-energy-prediction/train.csv",parse_dates=['timestamp'],dtype=train_dtype)
 test_dtype = {'meter':"uint8",'building_id':'uint16'}
test_cols_to_read = ['building_id','meter','timestamp']
test = pd.read_csv(".. /input/ashrae-energy-prediction/test.csv",parse_dates=['timestamp'],usecols=test_cols_to_read,dtype=test_dtype )
param_outputs = []
FOLD_VALUES = [x for x in train_set['fold'].unique() ]
for fold in train_set['fold'].unique() :
trn_idx = train_set['fold']!=fold
val_idx = train_set['fold']==fold
X_train = train_set.loc[trn_idx,SEL_FTS].values
y_train = train_set.loc[trn_idx, LABEL].values
X_val = train_set.loc[val_idx,SEL_FTS].values
y_val = train_set.loc[val_idx, LABEL].values
optimizer = Optimizer('mse')
 lgb_params = optimizer.optimize()
param_outputs += [lgb_params]
lgb_params['random_state'] = 42
model_output = LGBMRegressor(**lgb_params)
 print(fold, lgb_params)
 model_output.fit(X_train, y_train)
 preds = model_output.predict(X_val)
 print(mae(y_val, preds))
param_headings = ['colsample_bytree','subsample', 'reg_alpha',
'max_depth', 'min_child_samples', 'n_estimators', 'random_state']
params_output_df = pd.DataFrame(columns=param_headings, index=FOLD_VALUES, data=0.0)
 for idx in params_output_df.index:
for h in param_headings:
params_output_df.at[idx, h] = param_outputs[idx][h]
print('saving CSV')
 params_output_df.to_csv('optuna_lgbm_params.csv', index=True)
 param_means = params_output_df.mean(axis=0)
 param_means<save_to_csv>
Submission = pd.DataFrame(test.index,columns=['row_id'] )
sample_submission['time_to_eruption'] = test_predictions
sample_submission.to_csv('submission.csv', index=False )<load_from_csv>
missing_weather = pd.DataFrame(weather_train.isna().sum() /len(weather_train),columns=["Weather_Train_Missing_Pct"])
 missing_weather["Weather_Test_Missing_Pct"] = weather_test.isna().sum() /len(weather_test)
 missing_weather
train = pd.read_csv("/kaggle/input/predict-volcanic-eruptions-ingv-oe/train.csv")
 train<sort_values>
metadata.isna().sum() /len(metadata )
sorted_train = train.sort_values("time_to_eruption",ascending=False)
 sorted_train<create_dataframe>
metadata['floor_count_isNa'] = metadata['floor_count'].isna().astype('uint8')
 metadata['year_built_isNa'] = metadata['year_built'].isna().astype('uint8')
 metadata.drop('floor_count',axis=1,inplace=True )
test_ = pd.DataFrame([os.path.basename(f)[:-4] for f in glob.glob('/kaggle/input/predict-volcanic-eruptions-ingv-oe/test/*')], columns=["segment_id"])
 test_<sort_values>
missing_train_test = pd.DataFrame(train.isna().sum() /len(train),columns=["Missing_Pct_Train"])
 missing_train_test["Missing_Pct_Test"] = test.isna().sum() /len(test)
 missing_train_test
display(sorted_train.iloc[[0,-1],:])
 max_id = 1923243961
min_id = 601524801<load_from_csv>
train['meter'].replace({0:"Electricity",1:"ChilledWater",2:"Steam",3:"HotWater"},inplace=True)
 test['meter'].replace({0:"Electricity",1:"ChilledWater",2:"Steam",3:"HotWater"},inplace=True )
pd.read_csv(f"/kaggle/input/predict-volcanic-eruptions-ingv-oe/train/{max_id}.csv" ).plot(figsize=(15,10),title="max time_to_eruption(probably normal)",subplots=True,ylim=(-10000,10000))
plt.show()
pd.read_csv(f"/kaggle/input/predict-volcanic-eruptions-ingv-oe/train/{min_id}.csv" ).plot(figsize=(15,10),title="min time_to_eruption(probably abnormal)",subplots=True,ylim=(-10000,10000))
plt.show()<load_from_csv>
print("There are {} unique Buildings in the training data".format(train['building_id'].nunique()))
display(pd.read_csv(f"/kaggle/input/predict-volcanic-eruptions-ingv-oe/train/{max_id}.csv" ).isnull().sum())
 display(pd.read_csv(f"/kaggle/input/predict-volcanic-eruptions-ingv-oe/train/{min_id}.csv" ).isnull().sum() )<define_variables>
train['building_id'].value_counts(dropna=False ).head(20 )
fs = ["_mean","_std","_max","_min","_mad","_skew","_kurt","_nunique",
"_quantile_05","_quantile_10","_quantile_30","_quantile_70","_quantile_90","_quantile_95",
"_fft_power_mean","_fft_power_std","_fft_power_min","_fft_power_max",
"_fft_power_sum_low","_fft_power_sum_middle","_fft_power_sum_high",
"_fft_power_mad","_fft_power_skew","_fft_power_kurt","_fft_power_nunique",
"_fft_power_quantile_05","_fft_power_quantile_10","_fft_power_quantile_30","_fft_power_quantile_70","_fft_power_quantile_90","_fft_power_quantile_95",
"_cross_0_count",
"_roll_mean_min","_roll_mean_max","_roll_dist_min","_roll_dist_max","_roll_dist_diff_min","_roll_dist_diff_max",
]
def extract(segment_id,dir_="train"):

 f = pd.read_csv(f"/kaggle/input/predict-volcanic-eruptions-ingv-oe/{dir_}/{segment_id}.csv")
 f.interpolate(axis=0,inplace=True)
 q005 = f.quantile(0.05,axis=0)
 q010 = f.quantile(0.1 ,axis=0)
 q030 = f.quantile(0.3 ,axis=0)
 q070 = f.quantile(0.7 ,axis=0)
 q090 = f.quantile(0.9 ,axis=0)
 q095 = f.quantile(0.95,axis=0)
 roll = f.rolling(500)
 roll_mean = roll.mean()
roll_max = roll.max()
roll_min = roll.min()
roll_dist = roll_max - roll_min
roll_dist_diff = roll_dist.diff()
fft = pd.DataFrame(np.fft.fft(f.fillna(0)) ,columns=f.columns ).abs().iloc[1:30001,:]
fft[f.iloc[1:30001,:].isnull() ] = np.nan
f005 = f.where(f < q005)
 f010 = f.where(f < q010)
 f030 = f.where(f < q030)
 f070 = f.where(f > q070)
 f090 = f.where(f > q090)
 f095 = f.where(f > q095)
 f_abs = f.abs()
f_abs_0250 = f_abs.where(f_abs > 250)
 f_abs_0500 = f_abs.where(f_abs > 500)
 f_abs_0750 = f_abs.where(f_abs > 750)
 f_abs_1000 = f_abs.where(f_abs > 1000)
 f_abs_1250 = f_abs.where(f_abs > 1250)
 f_abs_1500 = f_abs.where(f_abs > 1500)
 return pd.concat(( f.mean(axis=0 ).add_suffix("_mean"),
f.std(axis=0 ).add_suffix("_std"),
f.max(axis=0 ).add_suffix("_max"),
f.min(axis=0 ).add_suffix("_min"),
f.mad(axis=0 ).add_suffix("_mad"),
f.skew(axis=0 ).add_suffix("_skew"),
f.kurt(axis=0 ).add_suffix("_kurt"),
f.nunique(axis=0 ).add_suffix("_nunique"),
q005.add_suffix("_quantile_05"),
q010.add_suffix("_quantile_10"),
q030.add_suffix("_quantile_30"),
q070.add_suffix("_quantile_70"),
q090.add_suffix("_quantile_90"),
q095.add_suffix("_quantile_95"),
fft.mean(axis=0 ).add_suffix("_fft_power_mean"),
fft.std(axis=0 ).add_suffix("_fft_power_std"),
fft.min(axis=0 ).add_suffix("_fft_power_min"),
fft.max(axis=0 ).add_suffix("_fft_power_max"),
fft.iloc[:10000,:].sum(axis=0 ).add_suffix("_fft_power_sum_low"),
fft.iloc[10000:20000,:].sum(axis=0 ).add_suffix("_fft_power_sum_middle"),
fft.iloc[20000:,:].sum(axis=0 ).add_suffix("_fft_power_sum_high"),
fft.mad(axis=0 ).add_suffix("_fft_power_mad"),
fft.skew(axis=0 ).add_suffix("_fft_power_skew"),
fft.kurt(axis=0 ).add_suffix("_fft_power_kurt"),
fft.nunique(axis=0 ).add_suffix("_fft_power_nunique"),
fft.quantile(0.05,axis=0 ).add_suffix("_fft_power_quantile_05"),
fft.quantile(0.1,axis=0 ).add_suffix("_fft_power_quantile_10"),
fft.quantile(0.3,axis=0 ).add_suffix("_fft_power_quantile_30"),
fft.quantile(0.7,axis=0 ).add_suffix("_fft_power_quantile_70"),
fft.quantile(0.9,axis=0 ).add_suffix("_fft_power_quantile_90"),
fft.quantile(0.95,axis=0 ).add_suffix("_fft_power_quantile_95"), 
(( f * f.shift())< 0 ).sum(axis=0 ).add_suffix("_cross_0_count"),
roll_mean.min(axis=0 ).add_suffix("_roll_mean_min"),
roll_mean.max(axis=0 ).add_suffix("_roll_mean_max"),
roll_dist.min(axis=0 ).add_suffix("_roll_dist_min"),
roll_dist.max(axis=0 ).add_suffix("_roll_dist_max"),
roll_dist_diff.min(axis=0 ).add_suffix("_roll_dist_diff_min"),
roll_dist_diff.max(axis=0 ).add_suffix("_roll_dist_diff_max"),
f005.idxmin().add_suffix("_first_005"),
f005.idxmax().add_suffix("_last_005"),
f010.idxmin().add_suffix("_first_010"),
f010.idxmax().add_suffix("_last_010"),
f030.idxmin().add_suffix("_first_030"),
f030.idxmax().add_suffix("_last_030"),
f070.idxmin().add_suffix("_first_070"),
f070.idxmax().add_suffix("_last_070"),
f090.idxmin().add_suffix("_first_090"),
f090.idxmax().add_suffix("_last_090"),
f095.idxmin().add_suffix("_first_095"),
f095.idxmax().add_suffix("_last_095"),
f_abs_0250.idxmin().add_suffix("_abs_0250_min"),
f_abs_0250.idxmax().add_suffix("_abs_0250_max"),
f_abs_0500.idxmin().add_suffix("_abs_0500_min"),
f_abs_0500.idxmax().add_suffix("_abs_0500_max"),
f_abs_0750.idxmin().add_suffix("_abs_0750_min"),
f_abs_0750.idxmax().add_suffix("_abs_0750_max"),
f_abs_1000.idxmin().add_suffix("_abs_1000_min"),
f_abs_1000.idxmax().add_suffix("_abs_1000_max"),
f_abs_1250.idxmin().add_suffix("_abs_1250_min"),
f_abs_1250.idxmax().add_suffix("_abs_1250_max"),
f_abs_1500.idxmin().add_suffix("_abs_1500_min"),
f_abs_1500.idxmax().add_suffix("_abs_1500_max"), 
),
axis=0 )<sort_values>
train.groupby('meter')['meter_reading'].agg(['min','max','mean','median','count','std'])
%time small_features = sorted_train.iloc[[0,1,2],:]["segment_id"].apply(extract)
 display(small_features)
 frame = small_features.iloc[:0]<save_to_csv>
for df in [train, test]:
df['Month'] = df['timestamp'].dt.month.astype("uint8")
 df['DayOfMonth'] = df['timestamp'].dt.day.astype("uint8")
 df['DayOfWeek'] = df['timestamp'].dt.dayofweek.astype("uint8")
 df['Hour'] = df['timestamp'].dt.hour.astype("uint8" )
%time features = dd.from_pandas(sorted_train["segment_id"],npartitions=4 ).apply(extract,meta=frame ).compute(scheduler="processes")
 data = pd.concat(( sorted_train,features),axis=1)
 data.to_csv("train_data.csv")
 data<save_to_csv>
train.groupby(['meter','Month'])['meter_reading'].agg(['max','mean','median','count','std'])
%time _test = dd.from_pandas(test_["segment_id"],npartitions=4 ).apply(extract,dir_="test",meta=frame ).compute(scheduler="processes")
 test = pd.concat(( test_,_test),axis=1)
 test.to_csv("test_data.csv")
 test<normalization>
train.groupby(['meter','DayOfWeek'])['meter_reading'].agg(['max','mean','median','count','std'])
scale_X = StandardScaler()
scale = StandardScaler()
train_X, test_X, train_y, test_y = train_test_split(data.drop(columns=["segment_id","time_to_eruption"]),
data[["time_to_eruption"]],
test_size=0.05)
 cols = [f"sensor_{i}{_fs}" for i in range(1,11)for _fs in fs]
scaled_train_X = pd.DataFrame(scale_X.fit_transform(train_X[cols]),index=train_X.index,columns=cols)
 scaled_test_X = pd.DataFrame(scale_X.transform(test_X[cols]),index=test_X.index ,columns=cols)
 scaled_test = pd.DataFrame(scale_X.transform(test[cols]),index=test.index ,columns=cols)
 scaled_train_y = scale.fit_transform(train_y)[:,0]
scaled_test_y = scale.transform(test_y)[:,0]
params = {'task' : 'train',
'boosting_type' : 'gbdt',
'objective' : 'regression',
'metric' : 'mae',
'verbose' : 0}
lgb_train = lgb.Dataset(scaled_train_X, scaled_train_y)
 lgb_eval = lgb.Dataset(scaled_test_X , scaled_test_y , reference=lgb_train)
 gbm = lgb.train(params,
lgb_train,
num_boost_round=1000,
valid_sets=[lgb_train, lgb_eval],
valid_names=["train","test"],
early_stopping_rounds=30)
 print(f"Best Iteration: {gbm.best_iteration}")
 print(f"Best Score: {gbm.best_score}")
 plt.figure(figsize=(12,7))
plt.plot(scaled_train_y,gbm.predict(scaled_train_X),marker=".",linestyle="",color="tab:blue",label="train data")
 plt.plot(scaled_test_y ,gbm.predict(scaled_test_X),marker=".",linestyle="",color="tab:red" ,label="test data")
 plt.plot(np.arange(-2.0,2.0,0.1),np.arange(-2.0,2.0,0.1),color="tab:green")
 plt.legend()
plt.xlabel("(Scaled)True time_to_eruption")
 plt.ylabel("(Scaled)Pred time_to_eruption")
 plt.show()
lgb.plot_importance(gbm,figsize=(10,70))
plt.show()<save_to_csv>
train.groupby('building_id')['meter_reading'].agg(['count','min','max','mean','median','std'])
submit = pd.DataFrame(scale.inverse_transform(gbm.predict(scaled_test)) ,
index=test["segment_id"],
columns=["time_to_eruption"],
dtype="int")
 submit.clip(lower=0,inplace=True)
 display(submit)
 submit.to_csv("submission.csv")
 !cat submission.csv<import_modules>
iqr = train[train['building_id'] == 1099]['meter_reading'].quantile(0.75)-train[train['building_id'] == 1099]['meter_reading'].quantile(0.25)
 q3 = train[train['building_id'] == 1099]['meter_reading'].quantile(0.75)
 q1 = train[train['building_id'] == 1099]['meter_reading'].quantile(0.25 )
import numpy as np
import pandas as pd
import lightgbm as lgb
import os<set_options>
outlier_range_building_1099 =(q1-1.5*iqr,q3+1.5*iqr)
 outlier_range_building_1099
print(h2o.__version__)
 h2o.init(max_mem_size='16G' )<load_from_csv>
cols = ['site_id','primary_use','building_id','year_built']
for col in cols:
print("Number of Unique Values in the {} column are:".format(col),metadata[col].nunique() )
train = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/train.csv')
 test = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv' )<load_from_csv>
metadata['primary_use'].value_counts(normalize=True)
scaled_feature_df = pd.read_csv('.. /input/ingv-tsfresh-7730/train.csv', sep = ';', index_col=0)
 scaled_feature_df = scaled_feature_df.loc[train['segment_id']]
scaled_test_df = pd.read_csv('.. /input/ingv-tsfresh-7730/test.csv', sep = ';', index_col=0)
 scaled_test_df = scaled_test_df.loc[test['segment_id']]<import_modules>
metadata['square_feet'] = np.log1p(metadata['square_feet'] )
from sklearn.feature_selection import SelectFromModel<train_model>
metadata.groupby('primary_use')['square_feet'].agg(['count','mean','median'] ).sort_values(by='count' )
sfm = SelectFromModel(estimator=lgb.LGBMRegressor())
 X = scaled_feature_df.drop('time_to_eruption',axis=1 ).copy()
X.columns = list(np.arange(len(X.columns)))
 y = scaled_feature_df['time_to_eruption']
sfm.fit(X, y )<drop_column>
metadata['year_built'].fillna(1976, inplace=True)
 metadata['year_built'] = metadata['year_built'].astype('int16' )
selected_features = list(scaled_feature_df.drop('time_to_eruption',axis=1 ).columns[sfm.get_support() ])
 selected_features<define_variables>
weather_train.isna().sum() /len(weather_train )
print('Number of selected features: ' + str(len(selected_features)) )<normalization>
def fill_weather_dataset(weather_df):
time_format = "%Y-%m-%d %H:%M:%S"
weather_df["datetime"] = pd.to_datetime(weather_df["timestamp"])
 weather_df["day"] = weather_df["datetime"].dt.day
weather_df["week"] = weather_df["datetime"].dt.week
weather_df["month"] = weather_df["datetime"].dt.month
weather_df = weather_df.set_index(['site_id','day','month'])
 air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean() ,columns=["air_temperature"])
 weather_df.update(air_temperature_filler,overwrite=False)
 cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()
cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=["cloud_coverage"])
 weather_df.update(cloud_coverage_filler,overwrite=False)
 due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean() ,columns=["dew_temperature"])
 weather_df.update(due_temperature_filler,overwrite=False)
 sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()
sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])
 weather_df.update(sea_level_filler,overwrite=False)
 wind_direction_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean() ,columns=['wind_direction'])
 weather_df.update(wind_direction_filler,overwrite=False)
 wind_speed_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean() ,columns=['wind_speed'])
 weather_df.update(wind_speed_filler,overwrite=False)
 precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()
precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])
 weather_df.update(precip_depth_filler,overwrite=False)
 weather_df = weather_df.reset_index()
weather_df = weather_df.drop(['datetime','day','week','month'],axis=1)
 return weather_df
scaled_feature_df = scaled_feature_df[selected_features]
scaled_test_df = scaled_test_df[selected_features]<create_dataframe>
weather_train = fill_weather_dataset(weather_train)
 weather_test = fill_weather_dataset(weather_test )
train_h2o = h2o.H2OFrame(scaled_feature_df)
 train_label_h2o = h2o.H2OFrame(train[['time_to_eruption']])
 train_h2o['time_to_eruption'] = train_label_h2o['time_to_eruption']
test_feature_h2o = h2o.H2OFrame(scaled_test_df )<prepare_x_and_y>
for df in [weather_train,weather_test]:
df['air_temperature'] = df['air_temperature'].astype('float32')
 df['cloud_coverage'] = df['cloud_coverage'].astype('float16')
 df['dew_temperature'] = df['dew_temperature'].astype('float16')
 df['precip_depth_1_hr'] = df['precip_depth_1_hr'].astype('float32')
 df['sea_level_pressure'] = df['sea_level_pressure'].astype('float32')
 df['wind_direction'] = df['wind_direction'].astype('float32')
 df['wind_speed'] = df['wind_speed'].astype('float16' )
x = test_feature_h2o.columns
y = 'time_to_eruption'<train_model>
%%time
train = pd.merge(train,metadata,on='building_id',how='left')
 test = pd.merge(test,metadata,on='building_id',how='left')
 print("Training Data Shape {}".format(train.shape))
print("Testing Data Shape {}".format(test.shape))
gc.collect()
aml = H2OAutoML(max_models=1000, seed=121, stopping_metric='MAE',
max_runtime_secs=360*60)
 aml.train(x=x, y=y, training_frame=train_h2o )<find_best_params>
%%time
train = pd.merge(train,weather_train,on=['site_id','timestamp'],how='left')
 test = pd.merge(test,weather_test,on=['site_id','timestamp'],how='left')
 print("Training Data Shape {}".format(train.shape))
print("Testing Data Shape {}".format(test.shape))
gc.collect()
aml.leader<predict_on_test>
for df in [train,test]:
df['square_feet'] = df['square_feet'].astype('float16')
 df['Age'] = df['timestamp'].dt.year - df['year_built']
df['Age_isNa'] = df['year_built_isNa']
preds = aml.predict(test_feature_h2o )<save_to_csv>
train.drop('year_built_isNa',axis=1,inplace=True)
 test.drop('year_built_isNa',axis=1,inplace=True )
submission = pd.DataFrame()
submission['segment_id'] = test['segment_id']
submission['time_to_eruption'] = preds.as_data_frame().values.flatten()
submission.loc[submission['time_to_eruption']<0, 'time_to_eruption'] = 0
submission.to_csv('submission_recent.csv', header=True, index=False )<import_modules>
idx_to_drop = list(( train[(train['site_id'] == 0)&(train['timestamp'] < "2016-05-21 00:00:00")] ).index)
 print(len(idx_to_drop))
train.drop(idx_to_drop,axis='rows',inplace=True )
import pandas as pd<load_from_csv>
idx_to_drop = list(train[(train['meter'] == "Electricity")&(train['meter_reading'] == 0)].index)
 print(len(idx_to_drop))
train.drop(idx_to_drop,axis='rows',inplace=True )
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 print(train.shape)
 train.head(5 )<load_from_csv>
idx_to_drop = list(( train[(train['building_id']==1099)&(train['meter_reading'] > 30000)&(train['meter'] == "Steam")] ).index)
 print(len(idx_to_drop))
train.drop(idx_to_drop,axis='rows',inplace=True )
test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 print(test.shape)
 test.head(5 )<import_modules>
train['meter_reading'] = np.log1p(train['meter_reading'] )
import re<feature_engineering>
%%time
mean_meter_reading_per_building = train.groupby('building_id')['meter_reading'].mean()
train['mean_meter_reading_per_building'] = train['building_id'].map(mean_meter_reading_per_building)
 median_meter_reading_per_building = train.groupby('building_id')['meter_reading'].median()
train['median_meter_reading_per_building'] = train['building_id'].map(median_meter_reading_per_building)
 std_meter_reading_per_building = train.groupby('building_id')['meter_reading'].std()
train['std_meter_reading_per_building'] = train['building_id'].map(std_meter_reading_per_building)
 mean_meter_reading_per_dayofweek = train.groupby('DayOfWeek')['meter_reading'].mean()
train['mean_meter_reading_per_dayofweek'] = train['DayOfWeek'].map(mean_meter_reading_per_dayofweek)
 median_meter_reading_per_dayofweek = train.groupby('DayOfWeek')['meter_reading'].median()
train['median_meter_reading_per_dayofweek'] = train['DayOfWeek'].map(median_meter_reading_per_dayofweek)
 std_meter_reading_per_dayofweek = train.groupby('DayOfWeek')['meter_reading'].std()
train['std_meter_reading_per_dayofweek'] = train['DayOfWeek'].map(std_meter_reading_per_dayofweek)
 mean_meter_reading_per_meter = train.groupby('meter')['meter_reading'].mean()
train['mean_meter_reading_per_meter'] = train['meter'].map(mean_meter_reading_per_meter)
 median_meter_reading_per_meter = train.groupby('meter')['meter_reading'].median()
train['median_meter_reading_per_meter'] = train['meter'].map(median_meter_reading_per_meter)
 std_meter_reading_per_meter = train.groupby('meter')['meter_reading'].std()
train['std_meter_reading_per_meter'] = train['meter'].map(std_meter_reading_per_meter)
 mean_meter_reading_per_month = train.groupby('Month')['meter_reading'].mean()
train['mean_meter_reading_per_month'] = train['Month'].map(mean_meter_reading_per_month)
 median_meter_reading_per_month = train.groupby('Month')['meter_reading'].median()
train['median_meter_reading_per_month'] = train['Month'].map(median_meter_reading_per_month)
 std_meter_reading_per_month = train.groupby('Month')['meter_reading'].std()
train['std_meter_reading_per_month'] = train['Month'].map(std_meter_reading_per_month)
 test['mean_meter_reading_per_building'] = test['building_id'].map(mean_meter_reading_per_building)
 test['median_meter_reading_per_building'] = test['building_id'].map(median_meter_reading_per_building)
 test['std_meter_reading_per_building'] = test['building_id'].map(std_meter_reading_per_building)
 test['mean_meter_reading_per_dayofweek'] = test['DayOfWeek'].map(mean_meter_reading_per_dayofweek)
 test['median_meter_reading_per_dayofweek'] = test['DayOfWeek'].map(median_meter_reading_per_dayofweek)
 test['std_meter_reading_per_dayofweek'] = test['DayOfWeek'].map(std_meter_reading_per_dayofweek)
 test['mean_meter_reading_per_meter'] = test['meter'].map(mean_meter_reading_per_meter)
 test['median_meter_reading_per_meter'] = test['meter'].map(median_meter_reading_per_meter)
 test['std_meter_reading_per_meter'] = test['meter'].map(std_meter_reading_per_meter)
 test['mean_meter_reading_per_month'] = test['Month'].map(mean_meter_reading_per_month)
 test['median_meter_reading_per_month'] = test['Month'].map(median_meter_reading_per_month)
 test['std_meter_reading_per_month'] = test['Month'].map(std_meter_reading_per_month )
def urlremove(text):
return re.sub(r"(?:\@|https?\://)\S+", "", text)
 train['text'] = train['text'].apply(urlremove)
 test['text'] = test['text'].apply(urlremove)
 train['text']<feature_engineering>
%%time
for df in [train, test]:
df['mean_meter_reading_per_building'] = df['mean_meter_reading_per_building'].astype("float16")
 df['median_meter_reading_per_building'] = df['mean_meter_reading_per_building'].astype("float16")
 df['std_meter_reading_per_building'] = df['std_meter_reading_per_building'].astype("float16")
 df['mean_meter_reading_per_meter'] = df['mean_meter_reading_per_meter'].astype("float16")
 df['median_meter_reading_per_meter'] = df['median_meter_reading_per_meter'].astype("float16")
 df['std_meter_reading_per_meter'] = df['std_meter_reading_per_meter'].astype("float16")
 df['mean_meter_reading_per_dayofweek'] = df['mean_meter_reading_per_dayofweek'].astype("float16")
 df['median_meter_reading_per_dayofweek'] = df['median_meter_reading_per_dayofweek'].astype("float16")
 df['std_meter_reading_per_dayofweek'] = df['std_meter_reading_per_dayofweek'].astype("float16")
 df['mean_meter_reading_per_month'] = df['mean_meter_reading_per_month'].astype("float16")
 df['median_meter_reading_per_month'] = df['median_meter_reading_per_month'].astype("float16")
 df['std_meter_reading_per_month'] = df['std_meter_reading_per_month'].astype("float16")
 df['Age'] = df['Age'].astype('uint8')
 gc.collect()
def remove_emoji(text):
emoji_pattern = re.compile(
 u"(\ud83d[\ude00-\ude4f])|"
u"(\ud83c[\udf00-\uffff])|"
u"(\ud83d[\u0000-\uddff])|"
u"(\ud83d[\ude80-\udeff])|"
u"(\ud83c[\udde0-\uddff])"
"+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 train['text'] = train['text'].apply(remove_emoji)
 test['text'] = test['text'].apply(remove_emoji)
 train['text']<categorify>
train.drop(['timestamp','year_built'],axis=1,inplace=True)
 test.drop(['timestamp','year_built'],axis=1,inplace=True )
def decontracted(text):
if '’' in text:
text = text.replace("’", "'")
 text = re.sub(r"n't", " not", text)
 text = re.sub(r"'re", " are", text)
 text = re.sub(r"'s", " is", text)
 text = re.sub(r"'d", " would", text)
 text = re.sub(r"'ll", " will", text)
 text = re.sub(r"'t", " not", text)
 text = re.sub(r"'ve", " have", text)
 text = re.sub(r"'m", " am", text)
 return text
train['text'] = train['text'].apply(decontracted)
 test['text'] = test['text'].apply(decontracted)
 train['text']<feature_engineering>
%%time
le = LabelEncoder()
train['meter']= le.fit_transform(train['meter'] ).astype("uint8")
 test['meter']= le.fit_transform(test['meter'] ).astype("uint8")
 train['primary_use']= le.fit_transform(train['primary_use'] ).astype("uint8")
 test['primary_use']= le.fit_transform(test['primary_use'] ).astype("uint8" )
def character_remove(text):
return re.sub('\W+|_', ' ', text)
 train.text = train.text.apply(character_remove)
 test.text = test.text.apply(character_remove)
 train.text<feature_engineering>
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
upper
def remove_numbers(text):
pattern = r'[^a-zA-z.,!?/:;"'\s]'
return re.sub(pattern, '', text)
 train.text = train.text.apply(remove_numbers)
 train.text = train.text.str.lower()
train.text = train.text.str.strip()
test.text = test.text.apply(remove_numbers)
 test.text = test.text.str.lower()
test.text = test.text.str.strip()
train['text']<import_modules>
to_drop = [column for column in upper.columns if any(upper[column] > threshold)]
print('There are %d columns to remove.' %(len(to_drop)))
 print("Following columns can be dropped {}".format(to_drop))
from sklearn.feature_extraction.text import TfidfVectorizer<feature_engineering>
train.drop(to_drop,axis=1,inplace=True)
 test.drop(to_drop,axis=1,inplace=True )
tfidf = TfidfVectorizer(min_df=1, max_df=0.3, ngram_range=(1, 2))
print(tfidf)
 text_tfidf = tfidf.fit_transform(train['text'])
 text_tfidf2 = tfidf.transform(test['text'])
<compute_train_metric>
y = train['meter_reading']
train.drop('meter_reading',axis=1,inplace=True )
clf = LogisticRegression(C=4.0)
 scores = model_selection.cross_val_score(clf, text_tfidf, train["target"], cv=3, scoring="f1")
 scores<train_model>
categorical_cols = ['building_id','Month','meter','Hour','primary_use','DayOfWeek','DayOfMonth','floor_count_isNa']
clf.fit(text_tfidf, train["target"] )<save_to_csv>
params = {'feature_fraction': 0.8,
'bagging_fraction': 0.5,
"bagging_freq": 5,
'objective': 'regression',
'max_depth': 11,
'learning_rate': 0.05,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'rmse',
"verbosity": -1,
'reg_alpha': 0.5,
'reg_lambda': 0.5,
'random_state': 47,
"num_leaves": 31}
kf = KFold(n_splits=3)
 models = []
for train_index,test_index in kf.split(train):
train_features = train.iloc[train_index]
train_target = y.iloc[train_index]
test_features = train.iloc[test_index]
test_target = y.iloc[test_index]
d_training = lgb.Dataset(train_features, label=train_target,categorical_feature=categorical_cols, free_raw_data=False)
 d_test = lgb.Dataset(test_features, label=test_target,categorical_feature=categorical_cols, free_raw_data=False)
 model = lgb.train(params, train_set=d_training, num_boost_round=2000, valid_sets=[d_training,d_test], verbose_eval=100, early_stopping_rounds=50)
 models.append(model)
 gc.collect()
submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 submission["target"] = clf.predict(text_tfidf2)
 submission.to_csv("submission-bylogistic.csv", index=False )<load_from_csv>
ser3 = pd.DataFrame(models[2].feature_importance() ,train.columns,columns=['Importance'] ).sort_values(by='Importance')
 ser3['Importance'].plot(kind='bar',figsize=(10,6))
train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train.head(10 )<count_values>
del train_features, train_target, test_features, test_target, d_training, d_test, weather_train, weather_test, metadata
train.count()<count_values>
stepsize = 500000
results = np.zeros(test.shape[0])
 for model in models:
predictions = []
for i in range(0, test.shape[0], stepsize):
predictions.append(np.expm1(model.predict(test.loc[i:i+stepsize-1,:], num_iteration=model.best_iteration)))
 results +=(1 / len(models)) * np.concatenate(predictions, axis=0)
 del(model )
test.count()<import_modules>
%%time
Submission['meter_reading'] = results
Submission['meter_reading'].clip(lower=0,upper=None,inplace=True)
 Submission.to_csv("Normal.csv",index=None )