Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<save_to_csv>
myfavouritenumber = 0
seed = myfavouritenumber
random.seed(seed)
 warnings.filterwarnings('ignore' )
test[['ID','Image','Diagnosis']] = test['ID'].str.split('_', expand=True)
 test['Image'] = 'ID_' + test['Image']
test = test[['Image', 'Label']]
test.drop_duplicates(inplace=True)
 test.to_csv('test.csv', index=False )<load_from_csv>
%%time
df_train = pd.read_feather('.. /input/ashrae-feather/train.ft')
 df_test = pd.read_feather('.. /input/ashrae-feather/test.ft')
 building = pd.read_feather('.. /input/ashrae-feather/building.ft')
 le = LabelEncoder()
building.primary_use = le.fit_transform(building.primary_use)
 DATA_PATH = ".. /input/ashrae-energy-prediction/"
weather_train = pd.read_csv(DATA_PATH + 'weather_train.csv')
 weather_test = weather_df = pd.read_csv(DATA_PATH + 'weather_test.csv' )
transform_train = Compose([
ShiftScaleRotate() ,
ToTensor()
])
 transform_test= Compose([
ToTensor()
])
 train_dataset = IntracranialDataset(
 csv_file='train.csv', path=dir_train_img, transform=transform_train, labels=True)
 test_dataset = IntracranialDataset(
 csv_file='test.csv', path=dir_test_img, transform=transform_test, labels=False)
 data_loader_train = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=False, num_workers=4)
 data_loader_test = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4 )<choose_model_class>
%%time
df_train = df_train.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20 18")')
 df_train = df_train.query('not(building_id == 681 & meter == 0 & timestamp <= "2016-04-27")')
 df_train = df_train.query('not(building_id == 761 & meter == 0 & timestamp <= "2016-09-02")')
 df_train = df_train.query('not(building_id == 799 & meter == 0 & timestamp <= "2016-09-02")')
 df_train = df_train.query('not(building_id == 802 & meter == 0 & timestamp <= "2016-08-24")')
 df_train = df_train.query('not(building_id == 1073 & meter == 0 & timestamp <= "2016-10-26")')
 df_train = df_train.query('not(building_id == 1094 & meter == 0 & timestamp <= "2016-09-08")')
 df_train = df_train.query('not(building_id == 29 & meter == 0 & timestamp <= "2016-08-10")')
 df_train = df_train.query('not(building_id == 40 & meter == 0 & timestamp <= "2016-06-04")')
 df_train = df_train.query('not(building_id == 45 & meter == 0 & timestamp <= "2016-07")')
 df_train = df_train.query('not(building_id == 106 & meter == 0 & timestamp <= "2016-11")')
 df_train = df_train.query('not(building_id == 107 & meter == 0 & timestamp >= "2016-11-10")')
 df_train = df_train.query('not(building_id == 112 & meter == 0 & timestamp < "2016-10-31 15")')
 df_train = df_train.query('not(building_id == 144 & meter == 0 & timestamp > "2016-05-14" & timestamp < "2016-10-31")')
 df_train = df_train.query('not(building_id == 147 & meter == 0 & timestamp > "2016-06-05 19" & timestamp < "2016-07-18 15")')
 df_train = df_train.query('not(building_id == 171 & meter == 0 & timestamp <= "2016-07-05")')
 df_train = df_train.query('not(building_id == 177 & meter == 0 & timestamp > "2016-06-04" & timestamp < "2016-06-25")')
 df_train = df_train.query('not(building_id == 258 & meter == 0 & timestamp > "2016-09-26" & timestamp < "2016-12-12")')
 df_train = df_train.query('not(building_id == 258 & meter == 0 & timestamp > "2016-08-30" & timestamp < "2016-09-08")')
 df_train = df_train.query('not(building_id == 258 & meter == 0 & timestamp > "2016-09-18" & timestamp < "2016-09-25")')
 df_train = df_train.query('not(building_id == 260 & meter == 0 & timestamp <= "2016-05-11")')
 df_train = df_train.query('not(building_id == 269 & meter == 0 & timestamp > "2016-06-04" & timestamp < "2016-06-25")')
 df_train = df_train.query('not(building_id == 304 & meter == 0 & timestamp >= "2016-11-20")')
 df_train = df_train.query('not(building_id == 545 & meter == 0 & timestamp > "2016-01-17" & timestamp < "2016-02-10")')
 df_train = df_train.query('not(building_id == 604 & meter == 0 & timestamp < "2016-11-21")')
 df_train = df_train.query('not(building_id == 693 & meter == 0 & timestamp > "2016-09-07" & timestamp < "2016-11-23")')
 df_train = df_train.query('not(building_id == 693 & meter == 0 & timestamp > "2016-07-12" & timestamp < "2016-05-29")')
 df_train = df_train.query('not(building_id == 723 & meter == 0 & timestamp > "2016-10-06" & timestamp < "2016-11-22")')
 df_train = df_train.query('not(building_id == 733 & meter == 0 & timestamp > "2016-05-29" & timestamp < "2016-06-22")')
 df_train = df_train.query('not(building_id == 733 & meter == 0 & timestamp > "2016-05-19" & timestamp < "2016-05-20")')
 df_train = df_train.query('not(building_id == 803 & meter == 0 & timestamp > "2016-9-25")')
 df_train = df_train.query('not(building_id == 815 & meter == 0 & timestamp > "2016-05-17" & timestamp < "2016-11-17")')
 df_train = df_train.query('not(building_id == 848 & meter == 0 & timestamp > "2016-01-15" & timestamp < "2016-03-20")')
 df_train = df_train.query('not(building_id == 857 & meter == 0 & timestamp > "2016-04-13")')
 df_train = df_train.query('not(building_id == 909 & meter == 0 & timestamp < "2016-02-02")')
 df_train = df_train.query('not(building_id == 909 & meter == 0 & timestamp < "2016-06-23")')
 df_train = df_train.query('not(building_id == 1008 & meter == 0 & timestamp > "2016-10-30" & timestamp < "2016-11-21")')
 df_train = df_train.query('not(building_id == 1113 & meter == 0 & timestamp < "2016-07-27")')
 df_train = df_train.query('not(building_id == 1153 & meter == 0 & timestamp < "2016-01-20")')
 df_train = df_train.query('not(building_id == 1169 & meter == 0 & timestamp < "2016-08-03")')
 df_train = df_train.query('not(building_id == 1170 & meter == 0 & timestamp > "2016-06-30" & timestamp < "2016-07-05")')
 df_train = df_train.query('not(building_id == 1221 & meter == 0 & timestamp < "2016-11-04")')
 df_train = df_train.query('not(building_id == 1225 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1234 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1233 & building_id <= 1234 & meter == 0 & timestamp > "2016-01-13 22" & timestamp < "2016-03-08 12")')
 df_train = df_train.query('not(building_id == 1241 & meter == 0 & timestamp > "2016-07-14" & timestamp < "2016-11-19")')
 df_train = df_train.query('not(building_id == 1250 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1255 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1264 & meter == 0 & timestamp > "2016-08-23")')
 df_train = df_train.query('not(building_id == 1265 & meter == 0 & timestamp > "2016-05-06" & timestamp < "2016-05-26")')
 df_train = df_train.query('not(building_id == 1272 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1275 & building_id <= 1280 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1283 & meter == 0 & timestamp > "2016-07-08" & timestamp < "2016-08-03")')
 df_train = df_train.query('not(building_id >= 1291 & building_id <= 1302 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1303 & meter == 0 & timestamp > "2016-07-25 22" & timestamp < "2016-07-27 16")')
 df_train = df_train.query('not(building_id == 1303 & meter == 0 & timestamp > "2016-01-26" & timestamp < "2016-06-02 12")')
 df_train = df_train.query('not(building_id == 1319 & meter == 0 & timestamp > "2016-05-17 16" & timestamp < "2016-06-07 12")')
 df_train = df_train.query('not(building_id == 1319 & meter == 0 & timestamp > "2016-08-18 14" & timestamp < "2016-09-02 14")')
 df_train = df_train.query('not(building_id == 1322 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp > "2016-10-14 22" & timestamp < "2016-10-17 08")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp > "2016-07-01 14" & timestamp < "2016-07-05 06")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp > "2016-10-14 22" & timestamp < "2016-10-17 08")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp > "2016-07-01 14" & timestamp < "2016-07-05 06")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp > "2016-10-14 22" & timestamp < "2016-10-17 08")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp > "2016-07-01 14" & timestamp < "2016-07-05 06")')
 df_train = df_train.query('not(building_id == 1272 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1291 & building_id <= 1297 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1300 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1302 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1291 & building_id <= 1299 & meter == 2 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1221 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1225 & building_id <= 1226 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1233 & building_id <= 1234 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1241 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1223 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1226 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1233 & building_id <= 1234 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1225 & building_id <= 1226 & meter == 2 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1305 & meter == 2 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1307 & meter == 2 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1223 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1231 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1233 & building_id <= 1234 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1272 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1275 & building_id <= 1297 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1300 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1302 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1293 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-25 12")')
 df_train = df_train.query('not(building_id == 1302 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-25 12")')
 df_train = df_train.query('not(building_id == 1223 & meter == 0 & timestamp > "2016-9-28 07" & timestamp < "2016-10-11 18")')
 df_train = df_train.query('not(building_id == 1225 & meter == 1 & timestamp > "2016-8-22 23" & timestamp < "2016-10-11 14")')
 df_train = df_train.query('not(building_id == 1230 & meter == 1 & timestamp > "2016-8-22 08" & timestamp < "2016-10-05 18")')
 df_train = df_train.query('not(building_id == 904 & meter == 0 & timestamp < "2016-02-17 08")')
 df_train = df_train.query('not(building_id == 986 & meter == 0 & timestamp < "2016-02-17 08")')
 df_train = df_train.query('not(building_id == 954 & meter == 0 & timestamp < "2016-08-08 11")')
 df_train = df_train.query('not(building_id == 954 & meter == 0 & timestamp < "2016-06-23 08")')
 df_train = df_train.query('not(building_id >= 745 & building_id <= 770 & meter == 1 & timestamp > "2016-10-05 01" & timestamp < "2016-10-10 09")')
 df_train = df_train.query('not(building_id >= 774 & building_id <= 787 & meter == 1 & timestamp > "2016-10-05 01" & timestamp < "2016-10-10 09")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp > "2016-05-11 09" & timestamp < "2016-05-12 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp > "2016-05-11 09" & timestamp < "2016-05-12 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp > "2016-05-11 09" & timestamp < "2016-05-12 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp == "2016-02-26 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp == "2016-02-26 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp == "2016-02-26 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp > "2016-03-29 10" & timestamp < "2016-03-30 12")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp > "2016-03-29 10" & timestamp < "2016-03-30 12")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp > "2016-03-29 10" & timestamp < "2016-03-30 12")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp > "2016-01-19 23" & timestamp < "2016-01-28 15")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp > "2016-01-19 23" & timestamp < "2016-01-28 15")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp > "2016-01-19 23" & timestamp < "2016-01-28 15")')
 df_train = df_train.query('not(building_id != 1227 & building_id != 1281 & building_id != 1314 & building_id >=1223 & building_id < 1335 & meter==0 & meter_reading==0)')
 df_train = df_train.query('not(building_id >= 1223 & building_id <= 1324 & meter==1 & timestamp > "2016-07-16 04" & timestamp < "2016-07-19 11")')
 df_train = df_train.query('not(building_id == 107 & meter == 0 & timestamp <= "2016-07-06")')
 df_train = df_train.query('not(building_id == 180 & timestamp >= "2016-02-17 12")')
 df_train = df_train.query('not(building_id == 182 & meter == 0)')
 df_train = df_train.query('not(building_id == 191 & meter == 0 & timestamp >= "2016-12-22 09")')
 df_train = df_train.query('not(building_id == 192 & meter == 1 & timestamp >= "2016-05-09 18")')
 df_train = df_train.query('not(building_id == 192 & meter == 3 & timestamp >= "2016-03-29 05" & timestamp <= "2016-04-04 08")')
 df_train = df_train.query('not(building_id == 207 & meter == 1 & timestamp > "2016-07-02 20" & timestamp < "2016-08-25 12")')
 df_train = df_train.query('not(building_id == 258 & timestamp > "2016-09-18" & timestamp < "2016-12-12 13")')
 df_train = df_train.query('not(building_id == 258 & timestamp > "2016-08-29 08" & timestamp < "2016-09-08 14")')
 df_train = df_train.query('not(building_id == 257 & meter == 1 & timestamp < "2016-03-25 16")')
 df_train = df_train.query('not(building_id == 260 & meter == 1 & timestamp > "2016-05-10 17" & timestamp < "2016-08-17 11")')
 df_train = df_train.query('not(building_id == 260 & meter == 1 & timestamp > "2016-08-28 01" & timestamp < "2016-10-31 13")')
 df_train = df_train.query('not(building_id == 220 & meter == 1 & timestamp > "2016-09-23 01" & timestamp < "2016-09-23 12")')
 df_train = df_train.query('not(building_id == 281 & meter == 1 & timestamp > "2016-10-25 08" & timestamp < "2016-11-04 15")')
 df_train = df_train.query('not(building_id == 273 & meter == 1 & timestamp > "2016-04-03 04" & timestamp < "2016-04-29 15")')
 df_train = df_train.query('not(building_id == 28 & meter == 0 & timestamp < "2016-10-14 20")')
 df_train = df_train.query('not(building_id == 71 & meter == 0 & timestamp < "2016-08-18 20")')
 df_train = df_train.query('not(building_id == 76 & meter == 0 & timestamp > "2016-06-04 09" & timestamp < "2016-06-04 14")')
 df_train = df_train.query('not(building_id == 101 & meter == 0 & timestamp > "2016-10-12 13" & timestamp < "2016-10-12 18")')
 df_train = df_train.query('not(building_id == 7 & meter == 1 & timestamp > "2016-11-03 09" & timestamp < "2016-11-28 14")')
 df_train = df_train.query('not(building_id == 9 & meter == 1 & timestamp > "2016-12-06 08")')
 df_train = df_train.query('not(building_id == 43 & meter == 1 & timestamp > "2016-04-03 08" & timestamp < "2016-06-06 13")')
 df_train = df_train.query('not(building_id == 60 & meter == 1 & timestamp > "2016-05-01 17" & timestamp < "2016-05-01 21")')
 df_train = df_train.query('not(building_id == 75 & meter == 1 & timestamp > "2016-08-05 13" & timestamp < "2016-08-26 12")')
 df_train = df_train.query('not(building_id == 95 & meter == 1 & timestamp > "2016-08-08 10" & timestamp < "2016-08-26 13")')
 df_train = df_train.query('not(building_id == 97 & meter == 1 & timestamp > "2016-08-08 14" & timestamp < "2016-08-25 14")')
 df_train = df_train.query('not(building_id == 1232 & meter == 1 & timestamp > "2016-06-23 16" & timestamp < "2016-08-31 20")')
 df_train = df_train.query('not(building_id == 1236 & meter == 1 & meter_reading >= 3000)')
 df_train = df_train.query('not(building_id == 1239 & meter == 1 & timestamp > "2016-03-11 16" & timestamp < "2016-03-27 17")')
 df_train = df_train.query('not(building_id == 1264 & meter == 1 & timestamp > "2016-08-22 17" & timestamp < "2016-09-22 20")')
 df_train = df_train.query('not(building_id == 1264 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1269 & meter == 1 & meter_reading >= 2000)')
 df_train = df_train.query('not(building_id == 1272 & meter == 1 & timestamp > "2016-08-11 12" & timestamp < "2016-08-30 19")')
 df_train = df_train.query('not(building_id == 1273 & meter == 1 & timestamp > "2016-05-31 14" & timestamp < "2016-06-17")')
 df_train = df_train.query('not(building_id == 1276 & meter == 1 & timestamp < "2016-02-03 23")')
 df_train = df_train.query('not(building_id == 1280 & meter == 1 & timestamp > "2016-05-18" & timestamp < "2016-05-26 09")')
 df_train = df_train.query('not(building_id == 1280 & meter == 1 & timestamp > "2016-02-28 23" & timestamp < "2016-05-02 05")')
 df_train = df_train.query('not(building_id == 1280 & meter == 1 & timestamp > "2016-06-12 01" & timestamp < "2016-7-07 06")')
 df_train = df_train.query('not(building_id == 1288 & meter == 1 & timestamp > "2016-07-07 15" & timestamp < "2016-08-12 17")')
 df_train = df_train.query('not(building_id == 1311 & meter == 1 & timestamp > "2016-04-25 18" & timestamp < "2016-05-13 14")')
 df_train = df_train.query('not(building_id == 1099 & meter == 2)')
 df_train = df_train.query('not(building_id == 1329 & meter == 0 & timestamp > "2016-04-28 00" & timestamp < "2016-04-28 07")')
 df_train = df_train.query('not(building_id == 1331 & meter == 0 & timestamp > "2016-04-28 00" & timestamp < "2016-04-28 07")')
 df_train = df_train.query('not(building_id == 1427 & meter == 0 & timestamp > "2016-04-11 10" & timestamp < "2016-04-11 14")')
 df_train = df_train.query('not(building_id == 1426 & meter == 2 & timestamp > "2016-05-03 09" & timestamp < "2016-05-03 14")')
 df_train = df_train.query('not(building_id == 1345 & meter == 0 & timestamp < "2016-03-01")')
 df_train = df_train.query('not(building_id == 1346 & timestamp < "2016-03-01")')
 df_train = df_train.query('not(building_id == 1359 & meter == 0 & timestamp > "2016-04-25 17" & timestamp < "2016-07-22 14")')
 df_train = df_train.query('not(building_id == 1365 & meter == 0 & timestamp > "2016-08-19 00" & timestamp < "2016-08-19 07")')
 df_train = df_train.query('not(building_id == 1365 & meter == 0 & timestamp > "2016-06-18 22" & timestamp < "2016-06-19 06")')
 df_train = df_train.query('not(building_id == 18 & meter == 0 & timestamp > "2016-06-04 09" & timestamp < "2016-06-04 16")')
 df_train = df_train.query('not(building_id == 18 & meter == 0 & timestamp > "2016-11-05 05" & timestamp < "2016-11-05 15")')
 df_train = df_train.query('not(building_id == 101 & meter == 0 & meter_reading > 800)')
 df_train = df_train.query('not(building_id == 1384 & meter == 0 & meter_reading == 0)')
 df_train = df_train.query('not(building_id >= 1289 & building_id <= 1301 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1243 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1263 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1284 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1286 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1263 & meter == 0 & timestamp > "2016-11-10 11" & timestamp < "2016-11-10 15")')
 df_train = df_train.query('not(building_id == 1238 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1329 & meter == 2 & timestamp > "2016-11-21 12" & timestamp < "2016-11-29 12")')
 df_train = df_train.query('not(building_id == 1249 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1250 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1256 & meter == 2 & timestamp > "2016-03-05 18" & timestamp < "2016-03-05 22")')
 df_train = df_train.query('not(building_id == 1256 & meter == 2 & timestamp > "2016-03-27 00" & timestamp < "2016-03-27 23")')
 df_train = df_train.query('not(building_id == 1256 & meter == 2 & timestamp > "2016-04-11 09" & timestamp < "2016-04-13 03")')
 df_train = df_train.query('not(building_id == 1256 & meter == 2 & timestamp > "2016-04-29 00" & timestamp < "2016-04-30 15")')
 df_train = df_train.query('not(building_id == 1303 & meter == 2 & timestamp < "2016-06-06 19")')
 df_train = df_train.query('not(building_id >= 1223 & building_id <= 1324 & meter == 1 & timestamp > "2016-08-11 17" & timestamp < "2016-08-12 17")')
 df_train = df_train.query('not(building_id >= 1223 & building_id <= 1324 & building_id != 1296 & building_id != 129 & building_id != 1298 & building_id != 1299 & meter == 2 & timestamp > "2016-08-11 17" & timestamp < "2016-08-12 17")')
 df_train = df_train.query('not(building_id >= 1223 & building_id <= 1324 & meter == 3 & timestamp > "2016-08-11 17" & timestamp < "2016-08-12 17")' )
device = torch.device("cuda:0")
 model = torch.hub.load('facebookresearch/WSL-Images', 'resnext101_32x8d_wsl')
 model.fc = torch.nn.Linear(2048, n_classes)
 model.to(device)
 criterion = torch.nn.BCEWithLogitsLoss()
plist = [{'params': model.parameters() , 'lr': 2e-5}]
optimizer = optim.Adam(plist, lr=2e-5)
 model, optimizer = amp.initialize(model, optimizer, opt_level="O1")
<train_model>
bm_cols = ['bm', 'weekday', 'hour',]
df_train['hour'] = df_train['timestamp'].dt.hour
df_train['weekday'] = df_train['timestamp'].dt.weekday
df_train['bm'] = df_train['building_id'].apply(lambda x: str(x)) + '_' + df_train['meter'].apply(lambda x: str(x))
bm = df_train.groupby(bm_cols)['meter_reading'].mean().rename('bm_week_hour' ).to_frame()
for epoch in range(n_epochs):
print('Epoch {}/{}'.format(epoch, n_epochs - 1))
print('-' * 10)
 model.train()
tr_loss = 0
tk0 = tqdm(data_loader_train, desc="Iteration")
 for step, batch in enumerate(tk0):
inputs = batch["image"]
labels = batch["labels"]
inputs = inputs.to(device, dtype=torch.float)
 labels = labels.to(device, dtype=torch.float)
 outputs = model(inputs)
 loss = criterion(outputs, labels)
 with amp.scale_loss(loss, optimizer)as scaled_loss:
scaled_loss.backward()
tr_loss += loss.item()
optimizer.step()
optimizer.zero_grad()
epoch_loss = tr_loss / len(data_loader_train)
 print('Training Loss: {:.4f}'.format(epoch_loss))<predict_on_test>
df_train = df_train.merge(bm, right_index=True, left_on=bm_cols, how='left' )
for param in model.parameters() :
param.requires_grad = False
model.eval()
test_pred = np.zeros(( len(test_dataset)* n_classes, 1))
for i, x_batch in enumerate(tqdm(data_loader_test)) :
x_batch = x_batch["image"]
x_batch = x_batch.to(device, dtype=torch.float)
 with torch.no_grad() :
pred = model(x_batch)
 test_pred[(i * batch_size * n_classes):(( i + 1)* batch_size * n_classes)] = torch.sigmoid(
 pred ).detach().cpu().reshape(( len(x_batch)* n_classes, 1))<save_to_csv>
df_train.drop(['bm'], axis=1, inplace=True)
 df_train.head()
submission = pd.read_csv(os.path.join(dir_csv, 'stage_1_sample_submission.csv'))
submission = pd.concat([submission.drop(columns=['Label']), pd.DataFrame(test_pred)], axis=1)
 submission.columns = ['ID', 'Label']
submission.to_csv('submission.csv', index=False)
 submission.head()<load_from_zip>
weather = pd.concat([weather_train, weather_test],ignore_index=True)
 weather_key = ['site_id', 'timestamp']
full_weather = weather[weather_key + ['air_temperature']].drop_duplicates(subset=weather_key ).sort_values(by=weather_key ).copy()
!rm -rf /kaggle/working/apex
!rm test.csv
!rm train.csv<define_variables>
full_weather['timestamp'] = pd.to_datetime(full_weather['timestamp'] )
dir_csv = '.. /input/rsna-intracranial-hemorrhage-detection'
dir_train_img = '.. /input/rsna-train-stage-1-images-png-224x/stage_1_train_png_224x'
dir_test_img = '.. /input/rsna-test-stage-1-images-png-224x/stage_1_test_png_224x' 
<define_variables>
full_weather['temp_rank'] = full_weather.groupby(['site_id', full_weather.timestamp.dt.date])['air_temperature'].rank('average')
 df_2d = full_weather.groupby(['site_id', full_weather.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)
 site_ids_offsets = pd.Series(df_2d.values.argmax(axis=1)- 14)
 site_ids_offsets.index.name = 'site_id'
def timestamp_align(df):
df['offset'] = df.site_id.map(site_ids_offsets)
 df['timestamp_aligned'] =(df.timestamp - pd.to_timedelta(df.offset, unit='H'))
df['timestamp'] = df['timestamp_aligned']
del df['timestamp_aligned']
return df
n_classes = 6
n_epochs = 5
batch_size = 128 
<install_modules>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df
!git clone https://github.com/NVIDIA/apex && cd apex && pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext"./
!pip install --upgrade efficientnet-pytorch 
<import_modules>
def fill_weather_dataset(weather_df):
time_format = "%Y-%m-%d %H:%M:%S"
start_date = datetime.datetime.strptime(weather_df['timestamp'].min() ,time_format)
 end_date = datetime.datetime.strptime(weather_df['timestamp'].max() ,time_format)
 total_hours = int(((end_date - start_date ).total_seconds() + 3600)/ 3600)
 hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format)for x in range(total_hours)]
missing_hours = []
for site_id in range(16):
site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])
 new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])
 new_rows['site_id'] = site_id
weather_df = pd.concat([weather_df,new_rows])
 weather_df = weather_df.reset_index(drop=True)
 weather_df["timestamp"] = pd.to_datetime(weather_df["timestamp"])
 weather_df["day"] = weather_df["timestamp"].dt.day
weather_df["week"] = weather_df["timestamp"].dt.week
weather_df["month"] = weather_df["timestamp"].dt.month
weather_df = weather_df.set_index(['site_id','day','month'])
 air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean() ,columns=["air_temperature"])
 weather_df.update(air_temperature_filler,overwrite=False)
 cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()
cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=["cloud_coverage"])
 weather_df.update(cloud_coverage_filler,overwrite=False)
 due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean() ,columns=["dew_temperature"])
 weather_df.update(due_temperature_filler,overwrite=False)
 sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()
sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])
 weather_df.update(sea_level_filler,overwrite=False)
 wind_direction_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean() ,columns=['wind_direction'])
 weather_df.update(wind_direction_filler,overwrite=False)
 wind_speed_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean() ,columns=['wind_speed'])
 weather_df.update(wind_speed_filler,overwrite=False)
 precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()
precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])
 weather_df.update(precip_depth_filler,overwrite=False)
 weather_df = weather_df.reset_index()
weather_df = weather_df.drop(['day','week','month'],axis=1)
 return weather_df
from apex import amp
import os
import cv2
import glob
import pydicom
import numpy as np
import pandas as pd
from efficientnet_pytorch import EfficientNet
import torch
import torch.optim as optim
from albumentations import Compose, ShiftScaleRotate, Resize
from albumentations.pytorch import ToTensor
from torch.utils.data import Dataset
from tqdm import tqdm_notebook as tqdm
from matplotlib import pyplot as plt<load_from_csv>
weather_train = fill_weather_dataset(weather_train )
class IntracranialDataset(Dataset):
def __init__(self, csv_file, path, labels, transform=None):
self.path = path
self.data = pd.read_csv(csv_file)
 self.transform = transform
self.labels = labels
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
img_name = os.path.join(self.path, self.data.loc[idx, 'Image'] + '.png')
 img = cv2.imread(img_name)
 if self.transform:
augmented = self.transform(image=img)
 img = augmented['image']
if self.labels:
labels = torch.tensor(
 self.data.loc[idx, ['epidural', 'intraparenchymal', 'intraventricular', 'subarachnoid', 'subdural', 'any']])
 return {'image': img, 'labels': labels}
else:
return {'image': img} 
<load_from_csv>
df_train = reduce_mem_usage(df_train, use_float16=True)
 building = reduce_mem_usage(building, use_float16=True)
 weather_train = reduce_mem_usage(weather_train, use_float16=True )
train = pd.read_csv(os.path.join(dir_csv, 'stage_1_train.csv'))
test = pd.read_csv(os.path.join(dir_csv, 'stage_1_sample_submission.csv'))<feature_engineering>
def rmse(ytrue, ypred):
return np.sqrt(np.mean(np.square(ypred - ytrue), axis=0))
def rmsle(ytrue, ypred):
return np.sqrt(np.mean(np.square(np.log1p(ypred)- np.log1p(ytrue)) , axis=0))
train[['ID', 'Image', 'Diagnosis']] = train['ID'].str.split('_', expand=True)
 train = train[['Image', 'Diagnosis', 'Label']]
train.drop_duplicates(inplace=True)
 train = train.pivot(index='Image', columns='Diagnosis', values='Label' ).reset_index()
train['Image'] = 'ID_' + train['Image']
train.head()<save_to_csv>
le = LabelEncoder()
def prepare_data(X, building_data, weather_data, test=False):

 weather_data = timestamp_align(weather_data)
 X = X.merge(building_data, on="building_id", how="left")
 X = X.merge(weather_data, on=["site_id", "timestamp"], how="left")
 X.sort_values("timestamp")
 X.reset_index(drop=True)
 gc.collect()
X.timestamp = pd.to_datetime(X.timestamp, format="%Y-%m-%d %H:%M:%S")
 X.square_feet = np.log1p(X.square_feet)
 X["hour"] = X.timestamp.dt.hour
X["weekday"] = X.timestamp.dt.weekday
X['hour'] = X['hour'].astype('int8')
 X['weekday'] = X['weekday'].astype('int8')
 drop_features = ["timestamp", "site_id","sea_level_pressure", "precip_depth_1_hr",]
X.drop(drop_features, axis=1, inplace=True)
 X = reduce_mem_usage(X, use_float16=True)
 if test:
row_ids = X.row_id
X.drop("row_id", axis=1, inplace=True)
 return X, row_ids
else:
y = np.log1p(X.meter_reading)
 X.drop("meter_reading", axis=1, inplace=True)
 return X, y
png = glob.glob(os.path.join(dir_train_img, '*.png'))
png = [os.path.basename(png)[:-4] for png in png]
png = np.array(png)
 train = train[train['Image'].isin(png)]
train.to_csv('train.csv', index=False )<save_to_csv>
X_train, y_train = prepare_data(df_train, building, weather_train)
 del df_train, weather_train
gc.collect()
test[['ID','Image','Diagnosis']] = test['ID'].str.split('_', expand=True)
 test['Image'] = 'ID_' + test['Image']
test = test[['Image', 'Label']]
test.drop_duplicates(inplace=True)
 test.to_csv('test.csv', index=False )<load_from_csv>
%%time
X_half_1 = X_train[:int(X_train.shape[0] / 2)]
X_half_2 = X_train[int(X_train.shape[0] / 2):]
y_half_1 = y_train[:int(X_train.shape[0] / 2)]
y_half_2 = y_train[int(X_train.shape[0] / 2):]
categorical_features = ["building_id", "meter", "primary_use", "hour", "weekday"]
d_half_1 = lgb.Dataset(X_half_1, label=y_half_1, categorical_feature=categorical_features, free_raw_data=False)
 d_half_2 = lgb.Dataset(X_half_2, label=y_half_2, categorical_feature=categorical_features, free_raw_data=False)
 watchlist_1 = [d_half_1, d_half_2]
watchlist_2 = [d_half_2, d_half_1]
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 40,
"learning_rate": 0.05,
"feature_fraction": 0.85,
"reg_lambda": 2,
"metric": "rmse",
"num_threads": 2
}
print("Building model with first half and validating on second half:")
 model_half_1 = lgb.train(params, train_set=d_half_1, num_boost_round=1000, valid_sets=watchlist_1, verbose_eval=200, early_stopping_rounds=200)
 print("Building model with second half and validating on first half:")
 model_half_2 = lgb.train(params, train_set=d_half_2, num_boost_round=1000, valid_sets=watchlist_2, verbose_eval=200, early_stopping_rounds=200 )
transform_train = Compose([
ShiftScaleRotate() ,
ToTensor()
])
 transform_test= Compose([
ToTensor()
])
 train_dataset = IntracranialDataset(
 csv_file='train.csv', path=dir_train_img, transform=transform_train, labels=True)
 test_dataset = IntracranialDataset(
 csv_file='test.csv', path=dir_test_img, transform=transform_test, labels=False)
 data_loader_train = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=False, num_workers=4)
 data_loader_test = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4 )<choose_model_class>
%%time
pred = np.zeros(len(X_train))
pred1 = model_half_1.predict(X_half_2)
 pred2 = model_half_2.predict(X_half_1 )
device = torch.device("cuda:0")
 model = EfficientNet.from_pretrained('efficientnet-b0')
 model._fc = torch.nn.Linear(1280, n_classes)
 model.to(device)
 criterion = torch.nn.BCEWithLogitsLoss()
plist = [{'params': model.parameters() , 'lr': 2e-5}]
optimizer = optim.Adam(plist, lr=2e-5)
 model, optimizer = amp.initialize(model, optimizer, opt_level="O1")
<train_model>
pred = np.zeros(len(X_train))
pred[:len(pred2)] += pred2
pred[len(pred2):] += pred1
print(f"Half/Half RMSE: {rmse(pred, y_train):.4f}" )
for epoch in range(n_epochs):
print('Epoch {}/{}'.format(epoch, n_epochs - 1))
print('-' * 10)
 model.train()
tr_loss = 0
tk0 = tqdm(data_loader_train, desc="Iteration")
 for step, batch in enumerate(tk0):
inputs = batch["image"]
labels = batch["labels"]
inputs = inputs.to(device, dtype=torch.float)
 labels = labels.to(device, dtype=torch.float)
 outputs = model(inputs)
 loss = criterion(outputs, labels)
 with amp.scale_loss(loss, optimizer)as scaled_loss:
scaled_loss.backward()
tr_loss += loss.item()
optimizer.step()
optimizer.zero_grad()
epoch_loss = tr_loss / len(data_loader_train)
 print('Training Loss: {:.4f}'.format(epoch_loss))<predict_on_test>
pickle.dump(model_half_1, open("model_half_1.pkl", "wb"))
pickle.dump(model_half_2, open("model_half_2.pkl", "wb"))
pickle.dump(pred, open("pred_L1.pkl", "wb"))
for param in model.parameters() :
param.requires_grad = False
model.eval()
test_pred = np.zeros(( len(test_dataset)* n_classes, 1))
for i, x_batch in enumerate(tqdm(data_loader_test)) :
x_batch = x_batch["image"]
x_batch = x_batch.to(device, dtype=torch.float)
 with torch.no_grad() :
pred = model(x_batch)
 test_pred[(i * batch_size * n_classes):(( i + 1)* batch_size * n_classes)] = torch.sigmoid(
 pred ).detach().cpu().reshape(( len(x_batch)* n_classes, 1))<save_to_csv>
del X_train, X_half_1, X_half_2, y_half_1, y_half_2, d_half_1, d_half_2,
del watchlist_1, watchlist_2, pred, pred1, pred2
gc.collect()
submission = pd.read_csv(os.path.join(dir_csv, 'stage_1_sample_submission.csv'))
submission = pd.concat([submission.drop(columns=['Label']), pd.DataFrame(test_pred)], axis=1)
 submission.columns = ['ID', 'Label']
submission.to_csv('submission.csv', index=False)
 submission.head()<load_from_zip>
weather_test = weather_df = pd.read_csv(DATA_PATH + 'weather_test.csv' )
!rm -rf /kaggle/working/apex
!rm test.csv
!rm train.csv<set_options>
%%time
df_test = pd.read_feather('.. /input/ashrae-feather/test.ft')
 weather_test = fill_weather_dataset(weather_test)
 df_test['hour'] = df_test['timestamp'].dt.hour
df_test['weekday'] = df_test['timestamp'].dt.weekday
df_test['bm'] = df_test['building_id'].apply(lambda x: str(x)) + '_' + df_test['meter'].apply(lambda x: str(x))
df_test = df_test.merge(bm, right_index=True, left_on=bm_cols, how='left')
 df_test.drop('bm', axis=1, inplace=True)
 df_test = reduce_mem_usage(df_test, use_float16=True)
 weather_test = reduce_mem_usage(weather_test, use_float16=True)
 X_test, row_ids = prepare_data(df_test, building, weather_test, test=True )
warnings.filterwarnings("ignore")
 pd.set_option('display.max_rows', 1000)
<load_from_csv>
del df_test, building, weather_test
gc.collect()
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )<define_variables>
%%time
pred = np.expm1(model_half_1.predict(X_test, num_iteration=model_half_1.best_iteration)) / 2
del model_half_1
gc.collect()
pred += np.expm1(model_half_2.predict(X_test, num_iteration=model_half_2.best_iteration)) / 2
del model_half_2
gc.collect()
num_col = train.loc[:,'MSSubClass':'SaleCondition'].select_dtypes(exclude=['object'] ).columns
def detect_outliers(df,n,features):

 outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(train,2, num_col)
 train.loc[Outliers_to_drop] 
<drop_column>
submission = pd.DataFrame({"row_id": row_ids, "meter_reading": np.clip(pred, 0, a_max=None)})
 submission['meter_reading'] = submission['meter_reading'].astype('float32')
 submission['row_id'] = submission['row_id'].astype('int32')
 submission.to_csv("submission.csv", index=False, chunksize=25000 )
train = train.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )<concatenate>
submission[X_test['building_id'] > 105]
df = pd.concat(( train.loc[:,'MSSubClass':'SaleCondition'], test.loc[:,'MSSubClass':'SaleCondition']))<feature_engineering>
site0 = pd.read_feather('.. /input/ucf-building-meter-reading/site0.ft')
 df_test = pd.read_feather('.. /input/ashrae-feather/test.ft' )
df['GrLivArea'] = df['GrLivArea'].clip(0, 3500)
 df['TotalBsmtSF'] = df['TotalBsmtSF'].clip(0, 3500)
 df['1stFlrSF'] = df['1stFlrSF'].clip(0, 3500)
 df['GarageArea'] = df['GarageArea'].clip(0, 1200)
 df['YearBuilt'] = df['YearBuilt'].clip(1950, df['YearBuilt'].max())
 df['YearRemodAdd'] = df['YearRemodAdd'].clip(1950, df['YearRemodAdd'].max())
 df['BsmtFinSF1'] = df['BsmtFinSF1'].clip(0, 2500)
 df['OpenPorchSF'] = df['OpenPorchSF'].clip(0, 400)
 df['LotArea'] = df['LotArea'].clip(0, 70000)
 df['TotRmsAbvGrd'] = df['TotRmsAbvGrd'].clip(0, 11)
 df['HalfBath'] = df['HalfBath'].clip(0, 1)
 df['BedroomAbvGr'] = df['BedroomAbvGr'].clip(0, 4)
 df['GarageYrBlt'] = df['GarageYrBlt'].clip(1950, df['GarageYrBlt'].max())
 df['Fireplaces'] = df['Fireplaces'].clip(0, 2)
 df['MasVnrArea'] = df['MasVnrArea'].clip(0, 1000)
 df['EnclosedPorch'] = df['EnclosedPorch'].clip(0, 400)
 df['ScreenPorch'] = df['ScreenPorch'].clip(0, 300 )<feature_engineering>
merged = df_test.merge(site0, left_on=['building_id', 'meter', 'timestamp'],
right_on=['building_id', 'meter', 'timestamp'], how='left' )
df['LowQualFinSF'] = np.log1p(df['LowQualFinSF'])
 df['LotArea'] = np.log1p(df['LotArea'])
 df['BsmtFinSF2'] = np.log1p(df['BsmtFinSF2'])
 df['ScreenPorch'] = np.log1p(df['ScreenPorch'])
 df['EnclosedPorch'] = np.log1p(df['EnclosedPorch'])
 df['MasVnrArea'] = np.log1p(df['MasVnrArea'])
 df['OpenPorchSF'] = np.log1p(df['OpenPorchSF'])
 df['WoodDeckSF'] = np.log1p(df['WoodDeckSF'])
 df['LotFrontage'] = np.log1p(df['LotFrontage'])
 df['1stFlrSF'] = np.log1p(df['1stFlrSF'] )<feature_engineering>
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']
def basic_details(df):
b = pd.DataFrame()
b['Missing value'] = df.isnull().sum()
b['N unique value'] = df.nunique()
b['dtype'] = df.dtypes
return b
basic_details(df )<normalization>
print(f'RMSLE of buildings 0-104: {rmsle(ytrue, pred):.4f}' )
scaler = MinMaxScaler(( -1,1))
df['BsmtFinSF2_sc'] = scaler.fit_transform(df[['BsmtFinSF2']])
 df['BsmtFinSF2_sc'].fillna(0,inplace=True)
 df['LowQualFinSF_sc'] = scaler.fit_transform(df[['LowQualFinSF']])
 df['WoodDeckSF_sc'] = scaler.fit_transform(df[['WoodDeckSF']])
 df['OpenPorchSF_sc'] = scaler.fit_transform(df[['OpenPorchSF']])
 df['EnclosedPorch_sc'] = scaler.fit_transform(df[['EnclosedPorch']])
 df['3SsnPorch_sc'] = scaler.fit_transform(df[['3SsnPorch']])
 df['ScreenPorch_sc'] = scaler.fit_transform(df[['ScreenPorch']])
 df['MiscVal_sc'] = scaler.fit_transform(df[['MiscVal']])
 df['MSZoning'].fillna('N')
 df['LotFrontage'].fillna(df['LotFrontage'].median() , inplace = True)
 df['Alley'].fillna('N')
 df['Exterior1st'].fillna('N')
 df['Exterior2nd'].fillna('N')
 df['Utilities'].fillna('N')
 df['MasVnrType'].fillna('N')
 df['BsmtFullBath'].fillna(0)
 df['BsmtHalfBath'].fillna(0)
 df['FullBath'].fillna(0)
 df['HalfBath'].fillna(0)
 df['KitchenQual'].fillna('N')
 df['Functional'].fillna('N')
 df['FireplaceQu'].fillna('N')
 df['GarageType'].fillna('N')
 df['GarageYrBlt'].fillna(0,inplace=True)
 df['GarageFinish'].fillna('N')
 df['GarageCars'].fillna(0)
 df['GarageArea'].fillna(0,inplace=True)
 df['GarageQual'].fillna('N')
 df['GarageCond'].fillna('N')
 df['BsmtFinSF2'].fillna(0,inplace=True)
 df['MasVnrArea'].fillna(0,inplace=True)
 df['BsmtFinSF1'].fillna(0,inplace=True)
 df['SaleType'].fillna('N')
 df['BsmtUnfSF'].fillna(0,inplace=True)
 df['TotalBsmtSF'].fillna(0,inplace=True)
 df['PoolQC'].fillna('N')
 df['Fence'].fillna('N')
 df['MiscFeature'].fillna('N')
 df['BsmtQual'].fillna('N')
 df['BsmtCond'].fillna('N')
 df['BsmtExposure'].fillna('N')
 df['BsmtFinType1'].fillna('N')
 df['BsmtFinType2'].fillna('N')
 df['Electrical'].fillna('N')
 df["AllSF"] = df["GrLivArea"] + df["TotalBsmtSF"]
df['Area'] = df['LotArea']*df['LotFrontage']
df['Area_log'] = np.log1p(df['Area'])
 def Gar_category(cat):
if cat <= 250:
return 1
elif cat <= 500 and cat > 250:
return 2
elif cat <= 1000 and cat > 500:
return 3
return 4
df['GarageArea_cat'] = df['GarageArea'].apply(Gar_category)
 def Low_category(cat):
if cat <= 1000:
return 1
elif cat <= 2000 and cat > 1000:
return 2
elif cat <= 3000 and cat > 2000:
return 3
return 4
df['GrLivArea_cat'] = df['GrLivArea'].apply(Low_category)
 def fl1_category(cat):
if cat <= 500:
return 1
elif cat <= 1000 and cat > 500:
return 2
elif cat <= 1500 and cat > 1000:
return 3
elif cat <= 2000 and cat > 1500:
return 4
return 5
df['1stFlrSF_cat'] = df['1stFlrSF'].apply(fl1_category)
 df['2ndFlrSF_cat'] = df['2ndFlrSF'].apply(fl1_category)
 def bsmtt_category(cat):
if cat <= 500:
return 1
elif cat <= 1000 and cat > 500:
return 2
elif cat <= 1500 and cat > 1000:
return 3
elif cat <= 2000 and cat > 1500:
return 4
return 5
df['TotalBsmtSF_cat'] = df['TotalBsmtSF'].apply(bsmtt_category)
 def bsmt_category(cat):
if cat <= 500:
return 1
elif cat <= 1000 and cat > 500:
return 2
elif cat <= 1500 and cat > 1000:
return 3
elif cat <= 2000 and cat > 1500:
return 4
return 5
df['BsmtUnfSF_cat'] = df['BsmtUnfSF'].apply(bsmt_category)
 def lot_category(cat):
if cat <= 50:
return 1
elif cat <= 100 and cat > 50:
return 2
elif cat <= 150 and cat > 100:
return 3
return 4
df['LotFrontage_cat'] = df['LotFrontage'].apply(lot_category)
 def lot_category1(cat):
if cat <= 5000:
return 1
elif cat <= 10000 and cat > 5000:
return 2
elif cat <= 15000 and cat > 10000:
return 3
elif cat <= 20000 and cat > 15000:
return 4
elif cat <= 25000 and cat > 20000:
return 5
return 6
df['LotArea_cat'] = df['LotArea'].apply(lot_category1)
 def year_category(yb):
if yb <= 1910:
return 1
elif yb <= 1950 and yb > 1910:
return 2
elif yb >= 1950 and yb < 1980:
return 3
elif yb >= 1980 and yb < 2000:
return 4
return 5
df['YearBuilt_cat'] = df['YearBuilt'].apply(year_category)
 df['YearRemodAdd_cat'] = df['YearRemodAdd'].apply(year_category)
 df['GarageYrBlt_cat'] = df['GarageYrBlt'].apply(year_category)
 def vnr_category(cat):
if cat <= 250:
return 1
elif cat <= 500 and cat > 250:
return 2
elif cat <= 750 and cat > 500:
return 3
return 4
df['MasVnrArea_cat'] = df['MasVnrArea'].apply(vnr_category)
 def allsf_category(yb):
if yb <= 1000:
return 1
elif yb <= 2000 and yb > 1000:
return 2
elif yb >= 3000 and yb < 2000:
return 3
elif yb >= 4000 and yb < 3000:
return 4
elif yb >= 5000 and yb < 4000:
return 5
elif yb >= 6000 and yb < 5000:
return 6
return 7
df['AllSF_cat'] = df['AllSF'].apply(allsf_category)
 dummy_col=['YearRemodAdd', 'YearBuilt', 'AllSF_cat', 'MiscVal','OverallCond', 'OverallQual', 'BsmtFinType2', 'SaleCondition','SaleType', 'YrSold', 'MoSold', 'MiscFeature', 'Fence', 'PoolQC', 'PoolArea', 'PavedDrive', 'GarageCond', 'GarageQual', 'GarageArea_cat', 'GarageCars', 'GarageFinish', 'GarageType', 'FireplaceQu', 'Fireplaces','Functional', 'TotRmsAbvGrd', 'KitchenQual', 'KitchenAbvGr', 'BedroomAbvGr', 'HalfBath', 'FullBath', 'BsmtHalfBath', 'BsmtFullBath','GrLivArea_cat','MSSubClass', 'MSZoning', 'LotFrontage_cat', 'LotArea_cat', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2',
'BldgType', 'HouseStyle', 'YearBuilt_cat', 'YearRemodAdd_cat', 'RoofStyle', 'RoofMatl', 'Exterior2nd', 'Exterior1st', 'MasVnrType', 'MasVnrArea_cat', 'ExterQual', 'ExterCond', 'Foundation',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtUnfSF_cat', 'TotalBsmtSF_cat', 'Heating', 'HeatingQC', 'CentralAir', 'Electrical', '1stFlrSF_cat', '2ndFlrSF_cat']
df = pd.get_dummies(df, columns=dummy_col, drop_first=False)
 df['LotFrontage_log'] = np.log1p(df['LotFrontage'])
 df['LotArea_log'] = np.log1p(df['LotArea'])
 df['BsmtUnfSF_log'] = np.log1p(df['BsmtUnfSF'])
 df['Is_MasVnr'] = [1 if i != 0 else 0 for i in df['MasVnrArea']]
df['Is_BsmtFinSF1'] = [1 if i != 0 else 0 for i in df['BsmtFinSF1']]
df['Is_BsmtFinSF2'] = [1 if i != 0 else 0 for i in df['BsmtFinSF2']]
df['Is_BsmtUnfSF'] = [1 if i != 0 else 0 for i in df['BsmtUnfSF']]
df['Is_TotalBsmtSF'] = [1 if i != 0 else 0 for i in df['TotalBsmtSF']]
df['Is_2ndFlrSF'] = [1 if i != 0 else 0 for i in df['2ndFlrSF']]
df['Is_LowQualFinSF'] = [1 if i != 0 else 0 for i in df['LowQualFinSF']]
df['Is_GarageArea'] = [1 if i != 0 else 0 for i in df['GarageArea']]
df['Is_WoodDeckSF'] = [1 if i != 0 else 0 for i in df['WoodDeckSF']]
df['Is_OpenPorchSF'] = [1 if i != 0 else 0 for i in df['OpenPorchSF']]
df['Is_EnclosedPorch'] = [1 if i != 0 else 0 for i in df['EnclosedPorch']]
df['Is_3SsnPorch'] = [1 if i != 0 else 0 for i in df['3SsnPorch']]
df['Is_ScreenPorch'] = [1 if i != 0 else 0 for i in df['ScreenPorch']] 
<feature_engineering>
site1 = pd.read_feather('.. /input/ucl-data-leakage-episode-2/site1.ft')
 site1 = site1.query('timestamp >= 2017' )
def basic_details(df):
b = pd.DataFrame()
b['Missing value'] = df.isnull().sum()
b['N unique value'] = df.nunique()
b['dtype'] = df.dtypes
return b
basic_details(df )<data_type_conversions>
merged = df_test.merge(site1, left_on=['building_id', 'meter', 'timestamp'],
right_on=['building_id', 'meter', 'timestamp'], how='left' )
def descrictive_stat_feat(df):
df = pd.DataFrame(df)
 dcol= [c for c in df.columns if df[c].nunique() >=10]
d_median = df[dcol].median(axis=0)
 d_mean = df[dcol].mean(axis=0)
 q1 = df[dcol].apply(np.float32 ).quantile(0.25)
 q3 = df[dcol].apply(np.float32 ).quantile(0.75)
 for c in dcol:
df[c+str('_median_range')] =(df[c].astype(np.float32 ).values > d_median[c] ).astype(np.int8)
 df[c+str('_mean_range')] =(df[c].astype(np.float32 ).values > d_mean[c] ).astype(np.int8)
 df[c+str('_q1')] =(df[c].astype(np.float32 ).values < q1[c] ).astype(np.int8)
 df[c+str('_q3')] =(df[c].astype(np.float32 ).values > q3[c] ).astype(np.int8)
 return df
df = descrictive_stat_feat(df )<prepare_x_and_y>
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']
X_train = df[:train.shape[0]]
X_test_fin = df[train.shape[0]:]
y = train.SalePrice
X_train['Y'] = y
df = X_train
X = df.drop('Y', axis=1)
 y = df.Y<init_hyperparams>
del merged, site1
print(f'RMSLE of buildings 105-155: {rmsle(ytrue, pred):.4f}' )
<init_hyperparams>
site2 = pd.read_feather('.. /input/asu-feather/site2.ft')
 site2 = site2.query('timestamp >= 2017' )
X = df.drop('Y', axis=1)
 y = df.Y
params = {
'objective':'reg:linear',
'n_estimators': 1000,
'booster':'gbtree',
'max_depth':2,
'eval_metric':'mae',
'learning_rate':0.1,
'min_child_weight':1,
'subsample':0.7,
'colsample_bytree':0.81,
'seed':45,
'reg_alpha':1e-05,
'gamma':0,
'nthread':-1
}
x_train, x_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=10)
 d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 d_test = xgb.DMatrix(X_test_fin)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
clf = xgb.train(params, d_train, 1200, watchlist, early_stopping_rounds=100, maximize=False, verbose_eval=10)
 p_test = clf.predict(d_test )<define_variables>
merged = df_test.merge(site2, left_on=['building_id', 'meter', 'timestamp'],
right_on=['building_id', 'meter', 'timestamp'], how='left' )
leaks = {
1461:105000,
1477:290941,
1492:67500,
1494:362500,
1514:84900,
1521:108538,
1531:80400,
1537:12789,
1540:76500,
1545:134000,
1554:122000,
1556:107500,
1557:100000,
1559:93369,
1560:114900,
1566:270000,
1567:85000,
1572:128000,
1573:308030,
1575:270000,
1586:84900,
1587:155891,
1589:64000,
1595:100000,
1597:215000,
1603:50138,
1610:174000,
1611:169000,
1615:76000,
1616:88250,
1617:85500,
1620:159000,
1622:161000,
1631:240000,
1638:154000,
1650:76500,
1652:111000,
1661:462000,
1664:610000,
1666:296000,
1678:552000,
1696:245000,
1698:327000,
1712:264500,
1717:152000,
1720:203000,
1726:171500,
1727:145000,
1737:275000,
1767:256000,
1774:135000,
1786:142900,
1787:156500,
1788:59000,
1790:78500,
1793:163000,
1807:103500,
1814:80000,
1820:58500,
1823:44000,
1831:179900,
1832:62500,
1835:97500,
1837:70000,
1842:63000,
1843:113500,
1863:269500,
1864:269500,
1892:85000,
1895:103500,
1912:315000,
1913:123000,
1915:230000,
1916:57625,
1925:170000,
1946:115000,
1947:334000,
1967:317500,
1970:390000,
1971:460000,
1975:615000,
1976:284000,
1996:284500,
1997:291000,
2004:297900,
2014:163000,
2030:300000,
2031:285000,
2032:290000,
2033:305000,
2038:345000,
2052:140000,
2055:141500,
2068:146000,
2076:94000,
2086:143000,
2093:122250,
2099:46500,
2100:65000,
2101:139500,
2106:55000,
2107:184000,
2111:108000,
2152:260000,
2162:475000,
2163:395039,
2180:185000,
2185:165000,
2206:104000,
2207:257076,
2208:263400,
2211:126000,
2217:13100,
2220:65000,
2223:300000,
2227:241500,
2230:172500,
2232:150000,
2235:195000,
2236:298751,
2238:209200,
2239:146000,
2245:94900,
2251:103000,
2263:349265,
2264:591587,
2267:441929,
2268:455000,
2269:174000,
2288:322400,
2295:500067,
2342:260000,
2354:146000,
2362:300000,
2375:279700,
2376:255000,
2379:240050,
2380:162500,
2395:224500,
2404:175000,
2419:115000,
2437:125500,
2455:136500,
2461:132000,
2465:165000,
2466:90000,
2468:113000,
2469:117000,
2474:50000,
2495:109900,
2544:110000,
2550:183850,
2557:79275,
2564:238000,
2565:153500,
2572:200000,
2574:315000,
2583:375000,
2590:244000,
2591:257000,
2599:392000,
2610:138000,
2611:80000,
2617:169000,
2618:252000,
2627:130000,
2631:535000,
2632:401179,
2634:470000,
2638:294323,
2658:344133,
2673:246990,
2690:405749,
2702:129500,
2723:157500,
2741:132000,
2752:167000,
2754:180000,
2760:80000,
2775:111500,
2776:156500,
2779:111500,
2788:64000,
2793:202500,
2794:75000,
2805:125000,
2813:156500,
2823:415000,
2829:224500,
2832:233555,
2859:98000,
2866:134000,
2872:35000,
2873:121000,
2881:195000,
2916:71000,
2917:131000,
2919:188000
}<save_to_csv>
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']
sub = pd.DataFrame()
sub['Id'] = test['Id']
sub['SalePrice'] = p_test
sub['SalePrice'] = sub.apply(lambda r: leaks[int(r['Id'])] if int(r['Id'])in leaks else r['SalePrice'], axis=1)
 sub.to_csv('submission.csv', index=False )<import_modules>
del site2, merged
print(f'RMSLE of buildings 156-290: {rmsle(ytrue, pred):.4f}' )
print('Python: {}'.format(sys.version))
print('scipy: {}'.format(scipy.__version__))
print('numpy: {}'.format(np.__version__))
print('matplotlib: {}'.format(matplotlib.__version__))
print('pandas: {}'.format(pd.__version__))
print('sklearn: {}'.format(sklearn.__version__))
print('seaborn: {}'.format(sns.__version__))<set_options>
site4 = pd.read_feather('.. /input/ucb-feather/site4.ft')
 site4 = site4.query('timestamp >= 2017' )
%matplotlib inline 
<load_from_csv>
merged = df_test.merge(site4, left_on=['building_id', 'timestamp'],
right_on=['building_id', 'timestamp'], how='left' )
df_train=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 df_test=pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )<count_missing_values>
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']
df_train.isnull().sum().sum() , df_test.isnull().sum().sum()<concatenate>
del site4, merged
print(f'RMSLE of 74/91 buildings : {rmsle(ytrue, pred):.4f}' )
all_features = pd.concat(( df_train.iloc[:, 1:-1], df_test.iloc[:, 1:]))<feature_engineering>
site15 = pd.read_feather('.. /input/cornell-feather/site15.ft')
 site15 = site15.query('timestamp >= 2017')
 site15 = site15.drop_duplicates()
numeric_features = all_features.dtypes[all_features.dtypes != 'object'].index
all_features[numeric_features] = all_features[numeric_features].apply(
 lambda x:(x - x.mean())/(x.std()))
all_features[numeric_features] = all_features[numeric_features].fillna(0 )<categorify>
merged = df_test.merge(site15, left_on=['building_id', 'meter', 'timestamp'],
right_on=['building_id', 'meter', 'timestamp'], how='left' )
all_features = pd.get_dummies(all_features, dummy_na=True)
 all_features.shape<count_missing_values>
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']
df_train.isnull().sum().sum() , df_test.isnull().sum().sum()<data_type_conversions>
del site15, merged
print(f'RMSLE of buildings 1325-1448: {rmsle(ytrue, pred):.4f}' )
n_train = df_train.shape[0]
train_features = np.array(all_features[:n_train].values, dtype=np.float32)
 test_features = np.array(all_features[n_train:].values, dtype=np.float32)
 train_labels = np.array(df_train.SalePrice.values.reshape(-1, 1), dtype=np.float32 )<prepare_x_and_y>
site012 = pd.read_feather('.. /input/comb-leaked-dataset/site012.ft')
 site012 = site012.query('timestamp >= 2017' )
y = train_labels
X = train_features
X_test = test_features<split>
merged = df_test.merge(site012, left_on=['building_id', 'meter', 'timestamp'],
right_on=['building_id', 'meter', 'timestamp'], how='left' )
train_X, val_X, train_y, val_y = train_test_split(X, y, random_state=1 )<choose_model_class>
ytrue = merged[~merged['meter_reading'].isna() ]['meter_reading']
pred = submission[~merged['meter_reading'].isna() ]['meter_reading']
kfold = KFold(n_splits=20, random_state= 0, shuffle = True )<predict_on_test>
del site012, merged
print(f'RMSLE of buildings 0-290: {rmsle(ytrue, pred):.4f}' )