Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<prepare_output>
%matplotlib inline
reg_sub = subm[["Patient_Week","FVC","Confidence"]].copy()<sort_values>
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 train_df = train_df [ train_df['building_id'] != 1099 ]
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")')
 building_df = pd.read_feather(root/'building_metadata.feather')
 weather_df = pd.read_feather(root/'weather_train.feather' )
df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True)
 df2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True ).reset_index(drop=True )<feature_engineering>
DATA_PATH = ".. /input/ashrae-energy-prediction/"
train_df_csv = pd.read_csv(DATA_PATH + 'train.csv')
 train_df_csv = train_df_csv[ train_df_csv['building_id'] != 1099 ]
train_df_csv = train_df_csv.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")')
 building_df_csv = pd.read_csv(DATA_PATH + 'building_metadata.csv')
 weather_df_csv = pd.read_csv(DATA_PATH + 'weather_train.csv' )
df = df1[['Patient_Week']].copy()
df['FVC'] = 0.25*df1['FVC'] + 0.75*df2['FVC']
df['Confidence'] = 0.25*df1['Confidence'] + 0.75*df2['Confidence']
df.head()<save_to_csv>
train_df['timestamp'] = train_df_csv['timestamp'].copy()
weather_df['timestamp'] = weather_df_csv['timestamp'].copy()
del weather_df_csv, train_df_csv
gc.collect()
df.to_csv('submission.csv', index=False )<import_modules>
def fill_weather(weather_df):
time_format = "%Y-%m-%d %H:%M:%S"
start_date = datetime.datetime.strptime(weather_df['timestamp'].min() ,time_format)
 end_date = datetime.datetime.strptime(weather_df['timestamp'].max() ,time_format)
 total_hours = int(((end_date - start_date ).total_seconds() + 3600)/ 3600)
 hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format)for x in range(total_hours)]
for site_id in range(16):
site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])
 missing_hour_data = pd.DataFrame(np.setdiff1d(hours_list, site_hours), columns=['timestamp'])
 missing_hour_data['site_id'] = site_id
weather_df = pd.concat([weather_df, missing_hour_data])
 weather_df.reset_index(drop=True)
 weather_df['datetime'] = pd.to_datetime(weather_df['timestamp'])
 weather_df['day'] = weather_df['datetime'].dt.day
weather_df['week'] = weather_df['datetime'].dt.week
weather_df['month'] = weather_df['datetime'].dt.month
weather_df = weather_df.set_index(['site_id', 'day', 'month'])
 for col in ['air_temperature', 'cloud_coverage', 'dew_temperature', 'sea_level_pressure', 'wind_direction', 'wind_speed',\
'precip_depth_1_hr']:
mean_filter = weather_df.groupby(['site_id', 'day', 'month'])[col].mean()
mean_filter = pd.DataFrame(mean_filter.fillna(method='ffill'), columns=[col])
 weather_df.update(mean_filter, overwrite=False)
 weather_df = weather_df.reset_index()
weather_df = weather_df.drop(['datetime', 'day', 'week', 'month'], axis=1)
 return weather_df
import pandas as pd
import numpy as np
import os
import tensorflow as tf<define_variables>
def feature_engineering(df):
df.sort_values('timestamp')
 df.reset_index(drop=True)
 df["timestamp"] = pd.to_datetime(df["timestamp"],format="%Y-%m-%d %H:%M:%S")
 df["hour"] = df["timestamp"].dt.hour
df["weekday"] = df["timestamp"].dt.weekday
df['square_feet'] = np.log1p(df['square_feet'])
 drop = ["timestamp","sea_level_pressure", "wind_direction", "wind_speed","year_built","floor_count"]
df = df.drop(drop, axis=1)
 gc.collect()
le = LabelEncoder()
df["primary_use"] = le.fit_transform(df["primary_use"])
 return df
img_size = 900<categorify>
weather_df = fill_weather(weather_df)
 train_df = reduce_mem_usage(train_df)
 building_df = reduce_mem_usage(building_df)
 weather_df = reduce_mem_usage(weather_df)
 train_df = train_df.merge(building_df, left_on='building_id',right_on='building_id',how='left')
 train_df = train_df.merge(weather_df,how='left',left_on=['site_id','timestamp'],right_on=['site_id','timestamp'])
 del weather_df
gc.collect()
train_df = feature_engineering(train_df )
def auto_select_accelerator() :
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("Running on TPU:", tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print(f"Running on {strategy.num_replicas_in_sync} replicas")
 return strategy
def build_decoder(with_labels=True, target_size=(img_size, img_size), ext='jpg'):
def decode(path):
file_bytes = tf.io.read_file(path)
 if ext == 'png':
img = tf.image.decode_png(file_bytes, channels=3)
 elif ext in ['jpg', 'jpeg']:
img = tf.image.decode_jpeg(file_bytes, channels=3)
 else:
raise ValueError("Image extension not supported")
 img = tf.cast(img, tf.float32)/ 255.0
img = tf.image.resize(img, target_size)
 return img
def decode_with_labels(path, label):
return decode(path), label
return decode_with_labels if with_labels else decode
def build_augmenter(with_labels=True):
def augment(img):
img = tf.image.random_flip_left_right(img)
 img = tf.image.random_flip_up_down(img)
 img = tf.image.random_saturation(img, 0.8, 1.2)
 img = tf.image.random_brightness(img, 0.1)
 img = tf.image.random_contrast(img, 0.9, 1.2)
 return img
def augment_with_labels(img, label):
return augment(img), label
return augment_with_labels if with_labels else augment
def build_dataset(paths, labels=None, bsize=32, cache=True,
decode_fn=None, augment_fn=None,
augment=True, repeat=True, shuffle=1024,
cache_dir=""):
if cache_dir != "" and cache is True:
os.makedirs(cache_dir, exist_ok=True)
 if decode_fn is None:
decode_fn = build_decoder(labels is not None)
 if augment_fn is None:
augment_fn = build_augmenter(labels is not None)
 AUTO = tf.data.experimental.AUTOTUNE
slices = paths if labels is None else(paths, labels)
 dset = tf.data.Dataset.from_tensor_slices(slices)
 dset = dset.map(decode_fn, num_parallel_calls=AUTO)
 dset = dset.cache(cache_dir)if cache else dset
dset = dset.map(augment_fn, num_parallel_calls=AUTO)if augment else dset
dset = dset.repeat() if repeat else dset
dset = dset.shuffle(shuffle)if shuffle else dset
dset = dset.batch(bsize ).prefetch(AUTO)
 return dset<define_variables>
train_X = train_df.drop(['meter_reading'], axis=1)
 train_y = np.log1p(train_df['meter_reading'])
 del train_df
gc.collect()
COMPETITION_NAME = "ranzcr-clip-catheter-line-classification"
strategy = auto_select_accelerator()
batch_size = strategy.num_replicas_in_sync * 16
print('batch size', batch_size )<load_pretrained>
categoricals = ['building_id', 'site_id', 'meter', 'primary_use', 'weekday']
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 1280,
"learning_rate": 0.05,
"feature_fraction": 0.85,
"reg_lambda": 2,
"metric": "rmse",
}
models = []
y_pred = np.zeros(( train_y.shape[0],))
kf = KFold(n_splits=3)
 for train_idx, valid_idx in kf.split(train_X):
train_features, train_target = train_X.loc[train_idx], train_y.loc[train_idx]
valid_features, valid_target = train_X.loc[valid_idx], train_y.loc[valid_idx]
lgb_train = lgb.Dataset(train_features, label=train_target, categorical_feature=categoricals, free_raw_data=False)
 lgb_valid = lgb.Dataset(valid_features, label=valid_target, categorical_feature=categoricals, free_raw_data=False)
 model = lgb.train(params, train_set=lgb_train, num_boost_round=1000, valid_sets=[lgb_train, lgb_valid],\
verbose_eval=25, early_stopping_rounds=50)
 models.append(model)
 y_pred[valid_idx] = model.predict(valid_features, num_iteration=model.best_iteration ).reshape(y_pred[valid_idx].shape)
 del train_features, train_target, valid_features, valid_target, lgb_train, lgb_valid
gc.collect()
print("final score(rmse): " + str(mean_squared_error(train_y, y_pred)) )
with strategy.scope() :
model = tf.keras.models.load_model('.. /input/tensorflow-data-dataset-tpu-part1/model.h5', custom_objects={'FixedDropout': tf.keras.layers.Dropout} )<load_from_csv>
del train_X, train_y, y_pred
gc.collect()
test_decoder = build_decoder(with_labels=False, target_size=(img_size, img_size))
load_dir = '/kaggle/input/ranzcr-clip-catheter-line-classification/'
df_sub = pd.read_csv(load_dir + 'sample_submission.csv')
 test_paths = load_dir + "test/" + df_sub['StudyInstanceUID'] + '.jpg'
dtest = build_dataset(
 test_paths, bsize=batch_size, repeat=False,
shuffle=False, augment=False, cache=False,
decode_fn=test_decoder
)<predict_on_test>
test_df = pd.read_feather(root/'test.feather')
 test_df_csv = pd.read_csv(DATA_PATH + 'test.csv')
 test_df['timestamp'] = test_df_csv['timestamp'].copy()
row_ids = test_df['row_id']
test_df = test_df.drop('row_id', axis=1)
 weather_df = pd.read_feather(root/'weather_test.feather')
 weather_df_csv = pd.read_csv(DATA_PATH + 'weather_test.csv')
 weather_df['timestamp'] = weather_df_csv['timestamp'].copy()
weather_df = fill_weather(weather_df)
 weather_df = reduce_mem_usage(weather_df)
 test_df = reduce_mem_usage(test_df)
 test_df = test_df.merge(building_df,left_on='building_id',right_on='building_id',how='left')
 test_df = test_df.merge(weather_df,how='left',on=['timestamp','site_id'])
 del weather_df, building_df, weather_df_csv, test_df_csv
gc.collect()
test_df = feature_engineering(test_df )
y_preds = model.predict(dtest, verbose=1 )<prepare_output>
results = []
for model in models:
if results == []:
results = np.expm1(model.predict(test_df, num_iteration=model.best_iteration)) / len(models)
 else:
results += np.expm1(model.predict(test_df, num_iteration=model.best_iteration)) / len(models)
 del model
gc.collect()
del test_df, models
gc.collect()
df_sub.iloc[:, 1:] = y_preds
display(df_sub )<save_to_csv>
submission = pd.DataFrame({"row_id": row_ids, "meter_reading": np.clip(results, 0, a_max=None)})
 del row_ids,results
gc.collect()
submission.to_csv("submission.csv", index=False )