Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<set_options>
warnings.filterwarnings('ignore')
 pd.set_option("max_columns", 500)
 %matplotlib inline
np.random.seed(1)
 nltk.download('stopwords')
 tf.random.set_seed(1)
 pd.set_option('display.max_colwidth', 500)
 warnings.filterwarnings('ignore' )<load_from_csv>
train = pd.read_csv(".. /input/ashrae-energy-prediction/train.csv", parse_dates=["timestamp"])
 test = pd.read_csv(".. /input/ashrae-energy-prediction/test.csv", parse_dates=["timestamp"])
 building = pd.read_csv('.. /input/ashrae-energy-prediction/building_metadata.csv')
 weather_train = pd.read_csv('.. /input/ashrae-energy-prediction/weather_train.csv', parse_dates=["timestamp"])
 weather_test = pd.read_csv(".. /input/ashrae-energy-prediction/weather_test.csv", parse_dates=["timestamp"] )
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 print("Train Shape :", train.shape)
 print("Test Shape :", test.shape )<count_missing_values>
train_plot = train.merge(building, on='building_id', how='left')
 site = 0
meter_type = 1
primary_use = 'Education'
r = int(np.ceil(len(train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)]['building_id'].value_counts(dropna=False ).index.to_list())/2))
fig, axes = plt.subplots(r,2,figsize=(14, 36), dpi=100)
 for i, building_id in enumerate(train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)]['building_id'].value_counts(dropna=False ).index.to_list()):
train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)& 
(train_plot['building_id'] == building_id)][['timestamp', 'meter_reading']].set_index('timestamp' ).resample('H' ).mean() ['meter_reading'].plot(ax=axes[i%r][i//r],
alpha=0.8, label='By hour', color='tab:blue' ).set_ylabel('Mean meter reading', fontsize=13);
train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)& 
(train_plot['building_id'] == building_id)][['timestamp', 'meter_reading']].set_index('timestamp' ).resample('D' ).mean() ['meter_reading'].plot(ax=axes[i%r][i//r],
alpha=1, label='By day', color='tab:orange' ).set_xlabel('');
axes[i%r][i//r].legend() ;
axes[i%r][i//r].set_title('building_id: ' + str(building_id), fontsize=13);
plt.subplots_adjust(hspace=0.45)
 del train_plot,fig,axes,r
gc.collect() ;
train.isnull().sum()<count_missing_values>
def compress_dataframe(df):

 result = df.copy()
for col in result.columns:
col_data = result[col]
dn = col_data.dtype.name
if dn == "object":
result[col] = pd.to_numeric(col_data.astype("category" ).cat.codes, downcast="integer")
 elif dn == "bool":
result[col] = col_data.astype("int8")
 elif dn.startswith("int")or(col_data.round() == col_data ).all() :
result[col] = pd.to_numeric(col_data, downcast="integer")
 else:
result[col] = pd.to_numeric(col_data, downcast='float')
 return result
train.isnull().sum()<count_values>
def set_time(df):
df.timestamp =(df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
return df
site_GMT_offsets = [-5, 0, -7, -5, -8, 0, -5, -5, -5, -6, -7, -5, 0, -6, -5, -5]
def weather_set_time(df,time_zone):
df.timestamp =(df.timestamp - pd.to_datetime("2016-01-01")).dt.total_seconds() // 3600
GMT_offset_map = {site: offset for site, offset in enumerate(site_GMT_offsets)}
df.timestamp = df.timestamp + df.site_id.map(GMT_offset_map)
 site_dfs = []
for site_id in df.site_id.unique() :
site_df = df[df.site_id == site_id].set_index("timestamp" ).reindex(time_zone)
 site_df.site_id = site_id
for col in [c for c in site_df.columns if c != "site_id"]:
site_df[f"had_{col}"] = ~site_df[col].isna()
site_df[col] = site_df[col].interpolate(limit_direction='both', method='linear')
 site_df[col] = site_df[col].fillna(df[col].median())
 site_dfs.append(site_df)
 df = pd.concat(site_dfs ).reset_index()
for col in df.columns:
if df[col].isna().any() : df[f"had_{col}"] = ~df[col].isna()
return df
def _add_time_features(X):
return X.assign(tm_day_of_week=(( X.timestamp // 24)% 7), tm_hour_of_day=(X.timestamp % 24))
building = compress_dataframe(building.fillna(-1)).set_index("building_id")
 train = compress_dataframe(set_time(train))
test = compress_dataframe(set_time(test)).set_index("row_id")
 weather_train = compress_dataframe(weather_set_time(weather_train,range(8784)) ).set_index(["site_id", "timestamp"])
 weather_test = compress_dataframe(weather_set_time(weather_test,range(8784,26304)) ).set_index(["site_id", "timestamp"] )
train['target'].value_counts(normalize = True )<count_values>
def combined_data(df,weather):
df = compress_dataframe(df.join(building, on="building_id" ).join(weather,
on=["site_id", "timestamp"] ).fillna(-1))
return df.drop(columns=["meter_reading"]),df.meter_reading
train.keyword.value_counts()<count_values>
def make_is_bad_zero(Xy_subset, min_interval=48, summer_start=3000, summer_end=7500):
meter = Xy_subset.meter_id.iloc[0]
is_zero = Xy_subset.meter_reading == 0
if meter == 0:
return is_zero
transitions =(is_zero != is_zero.shift(1))
all_sequence_ids = transitions.cumsum()
ids = all_sequence_ids[is_zero].rename("ids")
 if meter in [2, 3]:
keep = set(ids[(Xy_subset.timestamp < summer_start)| 
(Xy_subset.timestamp > summer_end)].unique())
 is_bad = ids.isin(keep)&(ids.map(ids.value_counts())>= min_interval)
 elif meter == 1:
time_ids = ids.to_frame().join(Xy_subset.timestamp ).set_index("timestamp" ).ids
is_bad = ids.map(ids.value_counts())>= min_interval
jan_id = time_ids.get(0, False)
 dec_id = time_ids.get(8283, False)
 if(jan_id and dec_id and jan_id == time_ids.get(500, False)and
dec_id == time_ids.get(8783, False)) :
is_bad = is_bad &(~(ids.isin(set([jan_id, dec_id]))))
else:
raise Exception(f"Unexpected meter type: {meter}")
 result = is_zero.copy()
result.update(is_bad)
 return result
def find_bad_zeros(X, y):

 Xy = X.assign(meter_reading=y, meter_id=X.meter)
 is_bad_zero = Xy.groupby(["building_id", "meter"] ).apply(make_is_bad_zero)
 return is_bad_zero[is_bad_zero].index.droplevel([0, 1])
 def find_bad_sitezero(X):

 return X[(X.timestamp < 3378)&(X.site_id == 0)&(X.meter == 0)].index
def find_bad_building1099(X, y):

 return X[(X.building_id == 1099)&(X.meter == 2)&(y > 3e4)].index
def find_bad_rows(X, y):
return find_bad_zeros(X, y ).union(find_bad_sitezero(X)).union(find_bad_building1099(X, y))
train.location.value_counts()<filter>
X, y = combined_data(train,weather_train)
 bad_rows = find_bad_rows(X, y)
 pd.Series(bad_rows.sort_values() ).to_csv("rows_to_drop.csv", header=False, index=False)
 X = X.drop(index=bad_rows)
 y = y.reindex_like(X)
 X = _add_time_features(X)
 X = compress_dataframe(X)
 X = X.drop(columns="timestamp")
 del bad_rows,train,weather_train
gc.collect() ;
real_tweets = train[train['target']==1]['text']
real_tweets.values[0:5]<define_variables>
y = np.log1p(y )
fake_tweets = train[train['target']==0]['text']
fake_tweets.values[0:5]<string_transform>
n_splits = 3
for val in X['meter'].unique() :
X1 = X[X['meter'] == val].drop(columns=['meter'])
 kf = StratifiedKFold(n_splits=n_splits,random_state=42)
 t = 0
for train_index, test_index in kf.split(X1, X1['tm_hour_of_day']):
train_features = X1.iloc[train_index]
train_target = y[X1.iloc[train_index].index]
test_features = X1.iloc[test_index]
test_target = y[X1.iloc[test_index].index]

 gbm = LGBMRegressor(
 boosting_type='gbdt',
objective='regression',
metric='rmse',
learning_rate=0.03,
num_leaves=40,
n_estimators=1000,
subsample=0.8 
)
 gbm.fit(train_features,train_target)
 joblib.dump(gbm, 'loan_model{}_fold{}.pkl'.format(val,t))
t += 1
del X1
del train_features, test_features
gc.collect() ;
def clean_text(text):
text = text.lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 return text<feature_engineering>
X = compress_dataframe(test.join(building, on="building_id" ).join(weather_test, on=["site_id", "timestamp"] ).fillna(-1))
X = compress_dataframe(_add_time_features(X))
X = X.drop(columns="timestamp")
 del test, weather_test
gc.collect() ;
<feature_engineering><EOS> 
result = np.zeros(len(X))
for val in X['meter'].unique() :
ix = np.nonzero(( X['meter'] == val ).to_numpy())
 for i in tqdm(range(n_splits)) :
gbm = joblib.load('loan_model{}_fold{}.pkl'.format(val,i))
result[ix] += gbm.predict(X.iloc[ix].drop(columns=['meter']), num_iteration=gbm.best_iteration_)/n_splits
gc.collect() ;
predictions = pd.DataFrame({
"row_id": X.index,
"meter_reading": np.clip(np.expm1(result), 0, None)
 })
 predictions.to_csv("submission.csv", index=False, float_format="%.4f" )