Kaggle competition: ASHRAE - Great Energy Predictor III
import matplotlib.pyplot as plt
import seaborn as sns
import pickle
from datetime import datetime as dt<load_from_csv>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df
df_input = pd.read_csv("/kaggle/input/nfl-big-data-bowl-2020/train.csv" )<create_dataframe>
train_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/train.csv')
 building_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/building_metadata.csv')
 weather_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_train.csv' )
df_all = df_input.copy()<define_variables>
personal_columms = ['X', 'Y', 'S', 'A', 'Dis', 'Orientation', 'Dir',
'NflId', 'DisplayName', 'JerseyNumber',
'PlayerHeight', 'PlayerWeight', 'PlayerBirthDate','PlayerCollegeName', 'Position']<set_options>
bad_rows = pd.read_csv('/kaggle/input/rows-to-drop/rows_to_drop.csv')
 train_df.drop(bad_rows.loc[:, '0'], inplace = True)
 train_df.reset_index(drop = True, inplace = True )
pd.set_option("display.max_columns", 80 )<import_modules>
def fill_weather_dataset(weather_df):
time_format = "%Y-%m-%d %H:%M:%S"
start_date = datetime.datetime.strptime(weather_df['timestamp'].min() ,time_format)
 end_date = datetime.datetime.strptime(weather_df['timestamp'].max() ,time_format)
 total_hours = int(((end_date - start_date ).total_seconds() + 3600)/ 3600)
 hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format)for x in range(total_hours)]
missing_hours = []
for site_id in range(16):
site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])
 new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])
 new_rows['site_id'] = site_id
weather_df = pd.concat([weather_df,new_rows])
 weather_df = weather_df.reset_index(drop=True)
 weather_df["datetime"] = pd.to_datetime(weather_df["timestamp"])
 weather_df["day"] = weather_df["datetime"].dt.day
weather_df["week"] = weather_df["datetime"].dt.week
weather_df["month"] = weather_df["datetime"].dt.month
weather_df = weather_df.set_index(['site_id','day','month'])
 air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean() ,columns=["air_temperature"])
 weather_df.update(air_temperature_filler,overwrite=False)
 cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()
cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=["cloud_coverage"])
 weather_df.update(cloud_coverage_filler,overwrite=False)
 due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean() ,columns=["dew_temperature"])
 weather_df.update(due_temperature_filler,overwrite=False)
 sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()
sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])
 weather_df.update(sea_level_filler,overwrite=False)
 wind_direction_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean() ,columns=['wind_direction'])
 weather_df.update(wind_direction_filler,overwrite=False)
 wind_speed_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean() ,columns=['wind_speed'])
 weather_df.update(wind_speed_filler,overwrite=False)
 precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()
precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])
 weather_df.update(precip_depth_filler,overwrite=False)
 weather_df = weather_df.reset_index()
weather_df = weather_df.drop(['datetime','day','week','month'],axis=1)
 def get_meteorological_features(data):
def calculate_rh(df):
df['relative_humidity'] = 100 *(np.exp(( 17.625 * df['dew_temperature'])/(243.04 + df['dew_temperature'])) / np.exp(( 17.625 * df['air_temperature'])/(243.04 + df['air_temperature'])))
 def calculate_fl(df):
flike_final = []
flike = []
for i in range(len(df)) :
at = df['air_temperature'][i]
rh = df['relative_humidity'][i]
ws = df['wind_speed'][i]
flike.append(feels_like(Temp(at, unit = 'C'), rh, ws))
for i in range(len(flike)) :
flike_final.append(flike[i].f)
 df['feels_like'] = flike_final
del flike_final, flike, at, rh, ws
calculate_rh(data)
 calculate_fl(data)
 return data
weather_df = get_meteorological_features(weather_df)
 return weather_df
import re<data_type_conversions>
def features_engineering(df):
df.sort_values("timestamp")
 df.reset_index(drop=True)
 df["timestamp"] = pd.to_datetime(df["timestamp"],format="%Y-%m-%d %H:%M:%S")
 df["hour"] = df["timestamp"].dt.hour
df["dayofweek"] = df["timestamp"].dt.dayofweek
df['month'] = df['timestamp'].dt.month
df['month'].replace(( 1, 2, 3, 4), 1, inplace = True)
 df['month'].replace(( 5, 6, 7, 8), 2, inplace = True)
 df['month'].replace(( 9, 10, 11, 12), 3, inplace = True)
 df['square_feet'] = np.log1p(df['square_feet'])
 drop = ["timestamp"]
df = df.drop(drop, axis=1)
 gc.collect()
le = LabelEncoder()
df["primary_use"] = le.fit_transform(df["primary_use"])
 return df
def is_float(s):
try:
float(s)
 return True
except ValueError:
return False<import_modules>
weather_df = fill_weather_dataset(weather_df)
 train_df = reduce_mem_usage(train_df,use_float16=True)
 building_df = reduce_mem_usage(building_df,use_float16=True)
 weather_df = reduce_mem_usage(weather_df,use_float16=True)
 train_df = train_df.merge(building_df, left_on='building_id',right_on='building_id',how='left')
 train_df = train_df.merge(weather_df,how='left',left_on=['site_id','timestamp'],right_on=['site_id','timestamp'])
 del weather_df
gc.collect()
re.split('[-/ ]', 'From SSW north'.replace("From ", "")) [0]<categorify>
train_df = features_engineering(train_df)
 train_df['meter_reading'] = np.log1p(train_df["meter_reading"] )
def preprocess(df):
outdoor=['Outdoor', 'Outdoors','Open','Indoor, Open Roof','Outdoor Retr Roof-Open', 'Oudoor',
'Ourdoor','Retr.Roof-Open','Outdor',
'Retr.Roof - Open', 'Domed, Open', 'Domed, open', 'Outside','Heinz Field']
indoor=['Indoors', 'RetractableRoof', 'Indoor','Retr.Roof-Closed','Dome', 'Domed, closed',
'Indoor, Roof Closed', 'Retr.Roof Closed','Closed Dome','Dome, closed','Domed']
df.loc[df['StadiumType'].isin(outdoor), 'stadiumtype']=1
df.loc[df['StadiumType'].isin(indoor), 'stadiumtype']=0
df.loc[:, "stadiumtype"] = df.loc[:, "stadiumtype"].fillna(2 ).astype(int)
 weather_keywords = [['indoor', 'controlled'],
['snow'],
['clear', 'sun', 'fair'],
['cloudy', 'overcast'],
['hazy'],
['rain', 'shower'],
]
df["weather"] = df["GameWeather"].apply(
 lambda x : min([ n for n,weather_class in enumerate(weather_keywords)
 for word in weather_class if word in str(x ).lower() ]+[len(weather_keywords)]))
df.loc[ df['StadiumType'].isin(indoor), "weather" ] = 0
natural_turf = ['Grass', 'Natural Grass', 'Natural grass', 'grass', 'Natural', 'Naturall Grass', 'natural grass']
df["natural_turf"] = df["Turf"].isin(natural_turf)
 df['PlayerHeight'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
df['GameClock']=[ pd.Timedelta(val ).total_seconds() for val in df['GameClock']]
df["TimeSnap"] = pd.to_datetime(df["TimeSnap"], format="%Y-%m-%dT%H:%M:%S.000Z")
 df = df.replace({'ARZ':'ARI', 'BLT':'BAL', 'CLV':'CLE', 'HST':'HOU'})
 df["WindSpeed"] = df["WindSpeed"].apply(lambda value :
np.mean([ float(oo)for oo in re.split('[^0-9.]+', str(value)) if is_float(oo)]))
df.loc[ df['StadiumType'].isin(indoor), "WindSpeed" ] = 0
df["yardsToTouchdown"] = df["YardLine"]
df.loc[ df["PossessionTeam"] == df["FieldPosition"], "yardsToTouchdown"] = 100-df["YardLine"]
return df<categorify>
target = 'meter_reading'
categorical = ['building_id', 'site_id', 'primary_use', 'meter','dayofweek']
numeric_cols = [col for col in train_df.columns if col not in categorical + [target, 'timestamp', 'month']]
features = categorical + numeric_cols
df_all = preprocess(df_all )<feature_engineering>
def run_lgbm(train, cat_features = categorical, num_rounds = 20000, folds = 5):
kf = StratifiedKFold(n_splits=folds, shuffle=False, random_state=2319)
 models = []
feature_importance_df = pd.DataFrame()
param = {'num_leaves': 3160,
'objective': 'regression',
'learning_rate': 0.03,
'boosting': 'gbdt',
'subsample': 0.5,
'feature_fraction': 0.7,
'n_jobs': -1,
'seed': 50,
'metric': 'rmse'
}
oof = np.zeros(len(train))
for tr_idx, val_idx in tqdm(kf.split(train_df,train_df['month']), total = folds):
tr_x, tr_y = train[features].iloc[tr_idx], train[target].iloc[tr_idx]
vl_x, vl_y = train[features].iloc[val_idx], train[target].iloc[val_idx]
tr_data = lgb.Dataset(tr_x, label = tr_y, categorical_feature = categorical)
 vl_data = lgb.Dataset(vl_x, label = vl_y, categorical_feature = categorical)
 clf = lgb.train(param, tr_data, num_rounds, valid_sets = [tr_data, vl_data], verbose_eval = 25,
early_stopping_rounds = 50)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 models.append(clf)
 oof[val_idx] = clf.predict(vl_x)
 gc.collect()
score = np.sqrt(metrics.mean_squared_error(train[target], np.clip(oof, a_min=0, a_max=None)))
 print('Our oof cv is :', score)
 cols =(feature_importance_df[["feature", "importance"]] 
.groupby("feature")
.mean() 
.sort_values(by="importance", ascending=False)[:20].index)
 best_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]
plt.figure(figsize=(14,26))
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance",ascending=False))
plt.title('LightGBM Features(averaged over folds)')
 plt.tight_layout()
plt.savefig('lgbm_importances.png')
 return models
models = run_lgbm(train_df )
def preprocess_coord(df):
df.loc[df["Season"]==2017, "Orientation"] =(df["Orientation"] -90)%360
df.loc[df["PlayDirection"]=="left", "X"] = 120 - df["X"]
df.loc[df["PlayDirection"]=="left", "Y"] = 53.3 - df["Y"]
df.loc[df["PlayDirection"]=="left", "Dir"] =(df["Dir"]+180)%360
df.loc[df["PlayDirection"]=="left", "Orientation"] =(df["Orientation"]+180)%360
df["X"] = df["X"] -(110-df["yardsToTouchdown"])
 df["Sy"] =(df["Dir"]*np.pi/180 ).map(np.cos)* df["S"]
df["Sx"] =(df["Dir"]*np.pi/180 ).map(np.sin)* df["S"]
df.drop(["S", "Dir"], axis=1, inplace=True)
 df["Orientation"] =(df["Orientation"]*np.pi/180 ).map(np.cos)
 return df<concatenate>
test_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/test.csv')
 row_ids = test_df["row_id"]
test_df.drop("row_id", axis=1, inplace=True)
 test_df = reduce_mem_usage(test_df)
 test_df = test_df.merge(building_df,left_on='building_id',right_on='building_id',how='left')
 del building_df
gc.collect()
weather_df = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_test.csv')
 weather_df = fill_weather_dataset(weather_df)
 weather_df = reduce_mem_usage(weather_df)
 test_df = test_df.merge(weather_df,how='left',on=['timestamp','site_id'])
 del weather_df
gc.collect()
test_df = features_engineering(test_df )
<filter><EOS> 
def predictions(models, iterations = 120):
set_size = len(test_df)
 batch_size = set_size // iterations
meter_reading = []
for i in tqdm(range(iterations)) :
pos = i*batch_size
fold_preds = [np.expm1(model.predict(test_df[features].iloc[pos : pos+batch_size])) for model in models]
meter_reading.extend(np.mean(fold_preds, axis=0))
print(len(meter_reading))
assert len(meter_reading)== set_size
submission = pd.read_csv('/kaggle/input/ashrae-energy-prediction/sample_submission.csv')
 submission['meter_reading'] = np.clip(meter_reading, a_min=0, a_max=None)
 submission.to_csv('fe2_lgbm.csv', index=False)
 print('We are done!')
 predictions(models )