Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<train_model>
py.init_notebook_mode(connected=True)
env = twosigmanews.make_env() 
(market_train_df, news_train_df)= env.get_training_data()
print('data loaded' )<filter>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
print('There are ', len(market_train_df['returnsOpenNextMktres10']), 'data before data selection')
 market_train_df = market_train_df.loc[market_train_df['time'] >= '2010-01-01 22:00:00+0000']
print('There are ', len(market_train_df['returnsOpenNextMktres10']), 'data after data selection' )<count_missing_values>
h2o.init(max_mem_size='14G')
 train = h2o.import_file(".. /input/ashrae-stack-set-train-fin/train.csv", header=1)
 y = "target"
x = train.columns[0:20]
glm_model = H2OGeneralizedLinearEstimator(
 family="gaussian",
solver='AUTO',
alpha=0.5,
link='Family_Default',
intercept=True,
lambda_search=True,
nlambdas=100,
missing_values_handling='MeanImputation',
standardize=True,
seed = 1333 
)
 glm_model.train(x=x, y=y, training_frame=train )
market_train_df.isna().sum()<drop_column>
h2o.remove(train)
 del train
gc.collect()
print('There are ', len(market_train_df['returnsOpenNextMktres10']), 'data before dropping NaN')
 market_train_df = market_train_df.dropna()
print('There are ', len(market_train_df['returnsOpenNextMktres10']), 'data after dropping NaN' )<count_missing_values>
test = h2o.import_file(".. /input/ashrae-stack-set-test-fin/test.csv", header=1)
 preds = glm_model.predict(test ).as_data_frame()
h2o.remove(test)
 del test
gc.collect()
h2o.cluster().shutdown()
market_train_df.isna().sum()<drop_column>
%%time
test_df = pd.read_csv('.. /input/ashrae-energy-prediction/test.csv', parse_dates=["timestamp"])
 building_meta_df = pd.read_csv('.. /input/ashrae-energy-prediction/building_metadata.csv' )
def remove_outlier_by_percentile(df,col_list,lower_p,upper_p):

 for i in range(len(col_list)) :
df =(df[(df[col_list[i]]<np.percentile(df[col_list[i]],upper_p)) &(df[col_list[i]]>np.percentile(df[col_list[i]],lower_p)) ])
 return df
outlier_removal_list = [ 'returnsClosePrevRaw1',
'returnsOpenPrevRaw1',
'returnsClosePrevRaw10',
'returnsOpenPrevRaw10',
'returnsOpenNextMktres10']
print('There are ', len(market_train_df['returnsOpenNextMktres10']), 'data before removing outliers')
 market_train_df = remove_outlier_by_percentile(market_train_df,outlier_removal_list,2,98)
 print('There are ', len(market_train_df['returnsOpenNextMktres10']), 'data after removing outliers' )<normalization>
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]
def scale_data(df,features):
scaler = StandardScaler()
df[features]=scaler.fit_transform(df[features])
 return df
features = ['volume','close','open','returnsClosePrevRaw1','returnsOpenPrevRaw1',
'returnsClosePrevMktres1','returnsOpenPrevMktres1','returnsClosePrevRaw10',
'returnsOpenPrevRaw10','returnsClosePrevMktres10','returnsOpenPrevMktres10', 'returnsOpenNextMktres10']
market_train_df = scale_data(market_train_df,features)
 market_train_df[features].head()<split>
test_df['pred'] = np.expm1(preds['predict'])
 test_df.loc[test_df.pred<0, 'pred'] = 0
test_df = reduce_mem_usage(test_df)
 leak_df = reduce_mem_usage(leak_df )
valid_ratio = 0.20
features = ['volume','close','open','returnsClosePrevRaw1','returnsOpenPrevRaw1',
'returnsClosePrevMktres1','returnsOpenPrevMktres1','returnsClosePrevRaw10',
'returnsOpenPrevRaw10','returnsClosePrevMktres10','returnsOpenPrevMktres10']
X = market_train_df[features]
Y = market_train_df['returnsOpenNextMktres10']
X_train, X_valid, Y_train, Y_valid = train_test_split(X,Y,test_size=valid_ratio)
<define_variables>
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred','row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']], on='building_id', how='left' )
input_dim = X_train.shape[1]
input_dim<train_on_grid>
sample_submission = pd.read_csv('.. /input/ashrae-energy-prediction/sample_submission.csv')
 sample_submission['meter_reading'] = test_df.pred
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
input_dim = X_train.shape[1]
output_dim = 1
dropout_rate = 0.2
sgd = optimizers.SGD(lr=0.01, clipnorm=1.)
model = Sequential()
model.add(Dense(11, input_dim=input_dim, activation = 'relu'))
model.add(Dense(6, activation = 'relu'))
model.add(Dropout(rate=dropout_rate))
model.add(Dense(output_dim))
model.compile(loss='mean_squared_error',optimizer=sgd)
 history = model.fit(X_train, Y_train,
epochs=20,
verbose=1,
validation_data =(X_valid, Y_valid),
callbacks=callbacks_list,
shuffle=True,
batch_size=128)
 print(model.summary())
<train_model>
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']
<create_dataframe><EOS> 
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )