Kaggle competition: Digit Recognizer
submission = pd.DataFrame({'ImageId' : range(1,28001), 'Label' : predictions} )<save_to_csv>
%matplotlib inline
submission.to_csv("submission.csv",index=False )<set_options>
train_df = pd.read_csv('.. /input/digit-recognizer/train.csv')
 n_pixels = len(train_df.columns)- 1
myfavouritenumber = 0
seed = myfavouritenumber
random.seed(seed)
 warnings.filterwarnings('ignore' )<load_from_csv>
BATCH_SIZE = 64
EPOCHS = 50
VALID_SIZE = 0.15
%%time
df_train = pd.read_feather('.. /input/ashrae-great-energy-predictor-iii-featherdataset/train.feather')
 df_test = pd.read_feather('.. /input/ashrae-great-energy-predictor-iii-featherdataset/test.feather')
 building = pd.read_feather('.. /input/ashrae-great-energy-predictor-iii-featherdataset/building_metadata.feather')
 le = LabelEncoder()
building.primary_use = le.fit_transform(building.primary_use)
 DATA_PATH = ".. /input/ashrae-energy-prediction/"
weather_train = pd.read_csv(DATA_PATH + 'weather_train.csv')
 weather_test = weather_df = pd.read_csv(DATA_PATH + 'weather_test.csv' )<feature_engineering>
class DatasetMNIST(torch.utils.data.Dataset):
def __init__(self, file_path,
transform = transforms.Compose([transforms.ToPILImage() , transforms.ToTensor() ,
transforms.Normalize(mean=(0.5,), std=(0.5,)) ])
):
df = pd.read_csv(file_path)
 if len(df.columns)== n_pixels:
self.X = df.values.reshape(( -1,28,28)).astype(np.uint8)[:,:,:,None]
self.y = None
else:
self.X = df.iloc[:,1:].values.reshape(( -1,28,28)).astype(np.uint8)[:,:,:,None]
self.y = torch.from_numpy(df.iloc[:,0].values)
 self.transform = transform
def __len__(self):
return len(self.X)
 def __getitem__(self, idx):
if self.y is not None:
return self.transform(self.X[idx]), self.y[idx]
else:
return self.transform(self.X[idx] )
%%time
df_train = df_train.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20 18")')
 df_train = df_train.query('not(building_id == 681 & meter == 0 & timestamp <= "2016-04-27")')
 df_train = df_train.query('not(building_id == 761 & meter == 0 & timestamp <= "2016-09-02")')
 df_train = df_train.query('not(building_id == 799 & meter == 0 & timestamp <= "2016-09-02")')
 df_train = df_train.query('not(building_id == 802 & meter == 0 & timestamp <= "2016-08-24")')
 df_train = df_train.query('not(building_id == 1073 & meter == 0 & timestamp <= "2016-10-26")')
 df_train = df_train.query('not(building_id == 1094 & meter == 0 & timestamp <= "2016-09-08")')
 df_train = df_train.query('not(building_id == 29 & meter == 0 & timestamp <= "2016-08-10")')
 df_train = df_train.query('not(building_id == 40 & meter == 0 & timestamp <= "2016-06-04")')
 df_train = df_train.query('not(building_id == 45 & meter == 0 & timestamp <= "2016-07")')
 df_train = df_train.query('not(building_id == 106 & meter == 0 & timestamp <= "2016-11")')
 df_train = df_train.query('not(building_id == 107 & meter == 0 & timestamp >= "2016-11-10")')
 df_train = df_train.query('not(building_id == 112 & meter == 0 & timestamp < "2016-10-31 15")')
 df_train = df_train.query('not(building_id == 144 & meter == 0 & timestamp > "2016-05-14" & timestamp < "2016-10-31")')
 df_train = df_train.query('not(building_id == 147 & meter == 0 & timestamp > "2016-06-05 19" & timestamp < "2016-07-18 15")')
 df_train = df_train.query('not(building_id == 171 & meter == 0 & timestamp <= "2016-07-05")')
 df_train = df_train.query('not(building_id == 177 & meter == 0 & timestamp > "2016-06-04" & timestamp < "2016-06-25")')
 df_train = df_train.query('not(building_id == 258 & meter == 0 & timestamp > "2016-09-26" & timestamp < "2016-12-12")')
 df_train = df_train.query('not(building_id == 258 & meter == 0 & timestamp > "2016-08-30" & timestamp < "2016-09-08")')
 df_train = df_train.query('not(building_id == 258 & meter == 0 & timestamp > "2016-09-18" & timestamp < "2016-09-25")')
 df_train = df_train.query('not(building_id == 260 & meter == 0 & timestamp <= "2016-05-11")')
 df_train = df_train.query('not(building_id == 269 & meter == 0 & timestamp > "2016-06-04" & timestamp < "2016-06-25")')
 df_train = df_train.query('not(building_id == 304 & meter == 0 & timestamp >= "2016-11-20")')
 df_train = df_train.query('not(building_id == 545 & meter == 0 & timestamp > "2016-01-17" & timestamp < "2016-02-10")')
 df_train = df_train.query('not(building_id == 604 & meter == 0 & timestamp < "2016-11-21")')
 df_train = df_train.query('not(building_id == 693 & meter == 0 & timestamp > "2016-09-07" & timestamp < "2016-11-23")')
 df_train = df_train.query('not(building_id == 693 & meter == 0 & timestamp > "2016-07-12" & timestamp < "2016-05-29")')
 df_train = df_train.query('not(building_id == 723 & meter == 0 & timestamp > "2016-10-06" & timestamp < "2016-11-22")')
 df_train = df_train.query('not(building_id == 733 & meter == 0 & timestamp > "2016-05-29" & timestamp < "2016-06-22")')
 df_train = df_train.query('not(building_id == 733 & meter == 0 & timestamp > "2016-05-19" & timestamp < "2016-05-20")')
 df_train = df_train.query('not(building_id == 803 & meter == 0 & timestamp > "2016-9-25")')
 df_train = df_train.query('not(building_id == 815 & meter == 0 & timestamp > "2016-05-17" & timestamp < "2016-11-17")')
 df_train = df_train.query('not(building_id == 848 & meter == 0 & timestamp > "2016-01-15" & timestamp < "2016-03-20")')
 df_train = df_train.query('not(building_id == 857 & meter == 0 & timestamp > "2016-04-13")')
 df_train = df_train.query('not(building_id == 909 & meter == 0 & timestamp < "2016-02-02")')
 df_train = df_train.query('not(building_id == 909 & meter == 0 & timestamp < "2016-06-23")')
 df_train = df_train.query('not(building_id == 1008 & meter == 0 & timestamp > "2016-10-30" & timestamp < "2016-11-21")')
 df_train = df_train.query('not(building_id == 1113 & meter == 0 & timestamp < "2016-07-27")')
 df_train = df_train.query('not(building_id == 1153 & meter == 0 & timestamp < "2016-01-20")')
 df_train = df_train.query('not(building_id == 1169 & meter == 0 & timestamp < "2016-08-03")')
 df_train = df_train.query('not(building_id == 1170 & meter == 0 & timestamp > "2016-06-30" & timestamp < "2016-07-05")')
 df_train = df_train.query('not(building_id == 1221 & meter == 0 & timestamp < "2016-11-04")')
 df_train = df_train.query('not(building_id == 1225 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1234 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1233 & building_id <= 1234 & meter == 0 & timestamp > "2016-01-13 22" & timestamp < "2016-03-08 12")')
 df_train = df_train.query('not(building_id == 1241 & meter == 0 & timestamp > "2016-07-14" & timestamp < "2016-11-19")')
 df_train = df_train.query('not(building_id == 1250 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1255 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1264 & meter == 0 & timestamp > "2016-08-23")')
 df_train = df_train.query('not(building_id == 1265 & meter == 0 & timestamp > "2016-05-06" & timestamp < "2016-05-26")')
 df_train = df_train.query('not(building_id == 1272 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1275 & building_id <= 1280 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1283 & meter == 0 & timestamp > "2016-07-08" & timestamp < "2016-08-03")')
 df_train = df_train.query('not(building_id >= 1291 & building_id <= 1302 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1303 & meter == 0 & timestamp > "2016-07-25 22" & timestamp < "2016-07-27 16")')
 df_train = df_train.query('not(building_id == 1303 & meter == 0 & timestamp > "2016-01-26" & timestamp < "2016-06-02 12")')
 df_train = df_train.query('not(building_id == 1319 & meter == 0 & timestamp > "2016-05-17 16" & timestamp < "2016-06-07 12")')
 df_train = df_train.query('not(building_id == 1319 & meter == 0 & timestamp > "2016-08-18 14" & timestamp < "2016-09-02 14")')
 df_train = df_train.query('not(building_id == 1322 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp > "2016-10-14 22" & timestamp < "2016-10-17 08")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp > "2016-07-01 14" & timestamp < "2016-07-05 06")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp > "2016-10-14 22" & timestamp < "2016-10-17 08")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp > "2016-07-01 14" & timestamp < "2016-07-05 06")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp > "2016-10-14 22" & timestamp < "2016-10-17 08")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp > "2016-07-01 14" & timestamp < "2016-07-05 06")')
 df_train = df_train.query('not(building_id == 1272 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1291 & building_id <= 1297 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1300 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1302 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1291 & building_id <= 1299 & meter == 2 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1221 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1225 & building_id <= 1226 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1233 & building_id <= 1234 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1241 & meter == 0 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1223 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1226 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1233 & building_id <= 1234 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1225 & building_id <= 1226 & meter == 2 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1305 & meter == 2 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1307 & meter == 2 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1223 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1231 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1233 & building_id <= 1234 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1272 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id >= 1275 & building_id <= 1297 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1300 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1302 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1293 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-25 12")')
 df_train = df_train.query('not(building_id == 1302 & meter == 3 & timestamp > "2016-09-28 07" & timestamp < "2016-10-25 12")')
 df_train = df_train.query('not(building_id == 1223 & meter == 0 & timestamp > "2016-9-28 07" & timestamp < "2016-10-11 18")')
 df_train = df_train.query('not(building_id == 1225 & meter == 1 & timestamp > "2016-8-22 23" & timestamp < "2016-10-11 14")')
 df_train = df_train.query('not(building_id == 1230 & meter == 1 & timestamp > "2016-8-22 08" & timestamp < "2016-10-05 18")')
 df_train = df_train.query('not(building_id == 904 & meter == 0 & timestamp < "2016-02-17 08")')
 df_train = df_train.query('not(building_id == 986 & meter == 0 & timestamp < "2016-02-17 08")')
 df_train = df_train.query('not(building_id == 954 & meter == 0 & timestamp < "2016-08-08 11")')
 df_train = df_train.query('not(building_id == 954 & meter == 0 & timestamp < "2016-06-23 08")')
 df_train = df_train.query('not(building_id >= 745 & building_id <= 770 & meter == 1 & timestamp > "2016-10-05 01" & timestamp < "2016-10-10 09")')
 df_train = df_train.query('not(building_id >= 774 & building_id <= 787 & meter == 1 & timestamp > "2016-10-05 01" & timestamp < "2016-10-10 09")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp > "2016-05-11 09" & timestamp < "2016-05-12 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp > "2016-05-11 09" & timestamp < "2016-05-12 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp > "2016-05-11 09" & timestamp < "2016-05-12 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp == "2016-02-26 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp == "2016-02-26 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp == "2016-02-26 01")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp > "2016-03-29 10" & timestamp < "2016-03-30 12")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp > "2016-03-29 10" & timestamp < "2016-03-30 12")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp > "2016-03-29 10" & timestamp < "2016-03-30 12")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 0 & timestamp > "2016-01-19 23" & timestamp < "2016-01-28 15")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 1 & timestamp > "2016-01-19 23" & timestamp < "2016-01-28 15")')
 df_train = df_train.query('not(building_id >= 874 & building_id <= 997 & meter == 2 & timestamp > "2016-01-19 23" & timestamp < "2016-01-28 15")')
 df_train = df_train.query('not(building_id != 1227 & building_id != 1281 & building_id != 1314 & building_id >=1223 & building_id < 1335 & meter==0 & meter_reading==0)')
 df_train = df_train.query('not(building_id >= 1223 & building_id <= 1324 & meter==1 & timestamp > "2016-07-16 04" & timestamp < "2016-07-19 11")')
 df_train = df_train.query('not(building_id == 107 & meter == 0 & timestamp <= "2016-07-06")')
 df_train = df_train.query('not(building_id == 180 & timestamp >= "2016-02-17 12")')
 df_train = df_train.query('not(building_id == 182 & meter == 0)')
 df_train = df_train.query('not(building_id == 191 & meter == 0 & timestamp >= "2016-12-22 09")')
 df_train = df_train.query('not(building_id == 192 & meter == 1 & timestamp >= "2016-05-09 18")')
 df_train = df_train.query('not(building_id == 192 & meter == 3 & timestamp >= "2016-03-29 05" & timestamp <= "2016-04-04 08")')
 df_train = df_train.query('not(building_id == 207 & meter == 1 & timestamp > "2016-07-02 20" & timestamp < "2016-08-25 12")')
 df_train = df_train.query('not(building_id == 258 & timestamp > "2016-09-18" & timestamp < "2016-12-12 13")')
 df_train = df_train.query('not(building_id == 258 & timestamp > "2016-08-29 08" & timestamp < "2016-09-08 14")')
 df_train = df_train.query('not(building_id == 257 & meter == 1 & timestamp < "2016-03-25 16")')
 df_train = df_train.query('not(building_id == 260 & meter == 1 & timestamp > "2016-05-10 17" & timestamp < "2016-08-17 11")')
 df_train = df_train.query('not(building_id == 260 & meter == 1 & timestamp > "2016-08-28 01" & timestamp < "2016-10-31 13")')
 df_train = df_train.query('not(building_id == 220 & meter == 1 & timestamp > "2016-09-23 01" & timestamp < "2016-09-23 12")')
 df_train = df_train.query('not(building_id == 281 & meter == 1 & timestamp > "2016-10-25 08" & timestamp < "2016-11-04 15")')
 df_train = df_train.query('not(building_id == 273 & meter == 1 & timestamp > "2016-04-03 04" & timestamp < "2016-04-29 15")')
 df_train = df_train.query('not(building_id == 28 & meter == 0 & timestamp < "2016-10-14 20")')
 df_train = df_train.query('not(building_id == 71 & meter == 0 & timestamp < "2016-08-18 20")')
 df_train = df_train.query('not(building_id == 76 & meter == 0 & timestamp > "2016-06-04 09" & timestamp < "2016-06-04 14")')
 df_train = df_train.query('not(building_id == 101 & meter == 0 & timestamp > "2016-10-12 13" & timestamp < "2016-10-12 18")')
 df_train = df_train.query('not(building_id == 7 & meter == 1 & timestamp > "2016-11-03 09" & timestamp < "2016-11-28 14")')
 df_train = df_train.query('not(building_id == 9 & meter == 1 & timestamp > "2016-12-06 08")')
 df_train = df_train.query('not(building_id == 43 & meter == 1 & timestamp > "2016-04-03 08" & timestamp < "2016-06-06 13")')
 df_train = df_train.query('not(building_id == 60 & meter == 1 & timestamp > "2016-05-01 17" & timestamp < "2016-05-01 21")')
 df_train = df_train.query('not(building_id == 75 & meter == 1 & timestamp > "2016-08-05 13" & timestamp < "2016-08-26 12")')
 df_train = df_train.query('not(building_id == 95 & meter == 1 & timestamp > "2016-08-08 10" & timestamp < "2016-08-26 13")')
 df_train = df_train.query('not(building_id == 97 & meter == 1 & timestamp > "2016-08-08 14" & timestamp < "2016-08-25 14")')
 df_train = df_train.query('not(building_id == 1232 & meter == 1 & timestamp > "2016-06-23 16" & timestamp < "2016-08-31 20")')
 df_train = df_train.query('not(building_id == 1236 & meter == 1 & meter_reading >= 3000)')
 df_train = df_train.query('not(building_id == 1239 & meter == 1 & timestamp > "2016-03-11 16" & timestamp < "2016-03-27 17")')
 df_train = df_train.query('not(building_id == 1264 & meter == 1 & timestamp > "2016-08-22 17" & timestamp < "2016-09-22 20")')
 df_train = df_train.query('not(building_id == 1264 & meter == 1 & timestamp > "2016-09-28 07" & timestamp < "2016-10-20 13")')
 df_train = df_train.query('not(building_id == 1269 & meter == 1 & meter_reading >= 2000)')
 df_train = df_train.query('not(building_id == 1272 & meter == 1 & timestamp > "2016-08-11 12" & timestamp < "2016-08-30 19")')
 df_train = df_train.query('not(building_id == 1273 & meter == 1 & timestamp > "2016-05-31 14" & timestamp < "2016-06-17")')
 df_train = df_train.query('not(building_id == 1276 & meter == 1 & timestamp < "2016-02-03 23")')
 df_train = df_train.query('not(building_id == 1280 & meter == 1 & timestamp > "2016-05-18" & timestamp < "2016-05-26 09")')
 df_train = df_train.query('not(building_id == 1280 & meter == 1 & timestamp > "2016-02-28 23" & timestamp < "2016-05-02 05")')
 df_train = df_train.query('not(building_id == 1280 & meter == 1 & timestamp > "2016-06-12 01" & timestamp < "2016-7-07 06")')
 df_train = df_train.query('not(building_id == 1288 & meter == 1 & timestamp > "2016-07-07 15" & timestamp < "2016-08-12 17")')
 df_train = df_train.query('not(building_id == 1311 & meter == 1 & timestamp > "2016-04-25 18" & timestamp < "2016-05-13 14")')
 df_train = df_train.query('not(building_id == 1099 & meter == 2)')
 df_train = df_train.query('not(building_id == 1329 & meter == 0 & timestamp > "2016-04-28 00" & timestamp < "2016-04-28 07")')
 df_train = df_train.query('not(building_id == 1331 & meter == 0 & timestamp > "2016-04-28 00" & timestamp < "2016-04-28 07")')
 df_train = df_train.query('not(building_id == 1427 & meter == 0 & timestamp > "2016-04-11 10" & timestamp < "2016-04-11 14")')
 df_train = df_train.query('not(building_id == 1426 & meter == 2 & timestamp > "2016-05-03 09" & timestamp < "2016-05-03 14")')
 df_train = df_train.query('not(building_id == 1345 & meter == 0 & timestamp < "2016-03-01")')
 df_train = df_train.query('not(building_id == 1346 & timestamp < "2016-03-01")')
 df_train = df_train.query('not(building_id == 1359 & meter == 0 & timestamp > "2016-04-25 17" & timestamp < "2016-07-22 14")')
 df_train = df_train.query('not(building_id == 1365 & meter == 0 & timestamp > "2016-08-19 00" & timestamp < "2016-08-19 07")')
 df_train = df_train.query('not(building_id == 1365 & meter == 0 & timestamp > "2016-06-18 22" & timestamp < "2016-06-19 06")')
 df_train = df_train.query('not(building_id == 18 & meter == 0 & timestamp > "2016-06-04 09" & timestamp < "2016-06-04 16")')
 df_train = df_train.query('not(building_id == 18 & meter == 0 & timestamp > "2016-11-05 05" & timestamp < "2016-11-05 15")')
 df_train = df_train.query('not(building_id == 101 & meter == 0 & meter_reading > 800)')
 df_train = df_train.query('not(building_id == 1384 & meter == 0 & meter_reading == 0)')
 df_train = df_train.query('not(building_id >= 1289 & building_id <= 1301 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1243 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1263 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1284 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1286 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1263 & meter == 0 & timestamp > "2016-11-10 11" & timestamp < "2016-11-10 15")')
 df_train = df_train.query('not(building_id == 1238 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1329 & meter == 2 & timestamp > "2016-11-21 12" & timestamp < "2016-11-29 12")')
 df_train = df_train.query('not(building_id == 1249 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1250 & meter == 2 & meter_reading == 0)')
 df_train = df_train.query('not(building_id == 1256 & meter == 2 & timestamp > "2016-03-05 18" & timestamp < "2016-03-05 22")')
 df_train = df_train.query('not(building_id == 1256 & meter == 2 & timestamp > "2016-03-27 00" & timestamp < "2016-03-27 23")')
 df_train = df_train.query('not(building_id == 1256 & meter == 2 & timestamp > "2016-04-11 09" & timestamp < "2016-04-13 03")')
 df_train = df_train.query('not(building_id == 1256 & meter == 2 & timestamp > "2016-04-29 00" & timestamp < "2016-04-30 15")')
 df_train = df_train.query('not(building_id == 1303 & meter == 2 & timestamp < "2016-06-06 19")')
 df_train = df_train.query('not(building_id >= 1223 & building_id <= 1324 & meter == 1 & timestamp > "2016-08-11 17" & timestamp < "2016-08-12 17")')
 df_train = df_train.query('not(building_id >= 1223 & building_id <= 1324 & building_id != 1296 & building_id != 129 & building_id != 1298 & building_id != 1299 & meter == 2 & timestamp > "2016-08-11 17" & timestamp < "2016-08-12 17")')
 df_train = df_train.query('not(building_id >= 1223 & building_id <= 1324 & meter == 3 & timestamp > "2016-08-11 17" & timestamp < "2016-08-12 17")' )<feature_engineering>
transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.RandomRotation(0, 0.5),
transforms.ToTensor() ,
transforms.Normalize(mean=(0.5,), std=(0.5,))
])
 train_data = DatasetMNIST('.. /input/digit-recognizer/train.csv', transform= transform)
 valid_train_data = DatasetMNIST('.. /input/digit-recognizer/train.csv', transform= transform)
 test_data = DatasetMNIST('.. /input/digit-recognizer/test.csv', transform= transform)
 num_train = len(train_data)
 indices = list(range(num_train))
np.random.shuffle(indices)
 split = int(np.floor(VALID_SIZE * num_train))
train_idx, valid_idx = indices[split:], indices[:split]
train_sampler = SubsetRandomSampler(train_idx)
 valid_sampler = SubsetRandomSampler(valid_idx)
 train_loader = torch.utils.data.DataLoader(dataset=train_data,
batch_size=BATCH_SIZE,sampler=train_sampler)
 valid_loader = torch.utils.data.DataLoader(dataset=valid_train_data,
batch_size=BATCH_SIZE,sampler=valid_sampler)
 test_loader = torch.utils.data.DataLoader(dataset=test_data,
batch_size=BATCH_SIZE, shuffle=False )
bm_cols = ['bm', 'weekday', 'hour',]
df_train['hour'] = df_train['timestamp'].dt.hour
df_train['weekday'] = df_train['timestamp'].dt.weekday
df_train['bm'] = df_train['building_id'].apply(lambda x: str(x)) + '_' + df_train['meter'].apply(lambda x: str(x))
bm = df_train.groupby(bm_cols)['meter_reading'].mean().rename('bm_week_hour' ).to_frame()<merge>
class CNN(nn.Module):
def __init__(self):
super(CNN,self ).__init__()
self.conv = nn.Sequential(
 nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(32),
nn.ReLU(inplace=True),
nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(32),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2),
nn.Dropout(p = 0.25),
nn.Conv2d(32, 64, kernel_size=3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.Conv2d(64, 64, kernel_size=3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2),
nn.Dropout(p = 0.25), 
)
 self.fc = nn.Sequential(
 nn.Linear(64 * 7 * 7, 512),
nn.BatchNorm1d(512),
nn.ReLU(inplace=True),
nn.Dropout(p = 0.5),
nn.Linear(512, 10), 
)
 for m in self.conv.children() :
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2./ n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
 m.bias.data.zero_()
for m in self.fc.children() :
if isinstance(m, nn.Linear):
nn.init.xavier_uniform_(m.weight)
 elif isinstance(m, nn.BatchNorm1d):
m.weight.data.fill_(1)
 m.bias.data.zero_()
def forward(self, x):
x= self.conv(x)
 x = x.view(x.size(0), -1)
 x = self.fc(x)
 return F.log_softmax(x, dim=1)
df_train = df_train.merge(bm, right_index=True, left_on=bm_cols, how='left' )<drop_column>
model = CNN()
print(model )
df_train.drop(['bm'], axis=1, inplace=True)
 df_train.head()<remove_duplicates>
optimizer = torch.optim.Adam(model.parameters() , lr=0.001)
 exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)
weather = pd.concat([weather_train, weather_test],ignore_index=True)
 weather_key = ['site_id', 'timestamp']
full_weather = weather[weather_key + ['air_temperature']].drop_duplicates(subset=weather_key ).sort_values(by=weather_key ).copy()<data_type_conversions>
error = nn.CrossEntropyLoss()
full_weather['timestamp'] = pd.to_datetime(full_weather['timestamp'] )<feature_engineering>
if torch.cuda.is_available() :
model = model.cuda()
error = error.cuda()
full_weather['temp_rank'] = full_weather.groupby(['site_id', full_weather.timestamp.dt.date])['air_temperature'].rank('average')
 df_2d = full_weather.groupby(['site_id', full_weather.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)
 site_ids_offsets = pd.Series(df_2d.values.argmax(axis=1)- 14)
 site_ids_offsets.index.name = 'site_id'
def timestamp_align(df):
df['offset'] = df.site_id.map(site_ids_offsets)
 df['timestamp_aligned'] =(df.timestamp - pd.to_timedelta(df.offset, unit='H'))
df['timestamp'] = df['timestamp_aligned']
del df['timestamp_aligned']
return df<set_options>
def fit(epoch):
valid_loss_min = np.Inf
running_loss= 0
accuracy_train = 0
model.train()
exp_lr_scheduler.step()
for batch_idx,(data, target)in enumerate(train_loader):
data, target = Variable(data), Variable(target)
 if torch.cuda.is_available() :
data = data.cuda()
target = target.cuda()
optimizer.zero_grad()
output = model(data)
 loss = error(output, target)
 loss.backward()
optimizer.step()
running_loss += loss.item()
_, top_class = output.topk(1, dim=1)
 equals = top_class == target.view(*top_class.shape)
 accuracy_train += torch.mean(equals.type(torch.FloatTensor))
valid_loss = 0
accuracy_val = 0
with torch.no_grad() :
model.eval()
for data, target in valid_loader:
data, target = Variable(data), Variable(target)
 if torch.cuda.is_available() :
data, target = data.cuda() , target.cuda()
output = model(data)
 _, top_class = output.topk(1, dim=1)
 equals = top_class == target.view(*top_class.shape)
 loss = error(output, target)
 valid_loss += loss.item()
accuracy_val += torch.mean(equals.type(torch.FloatTensor))
model.train()
train_losses.append(running_loss/len(train_loader))
valid_losses.append(valid_loss/len(valid_loader))
history_accuracy_val.append(accuracy_val/len(valid_loader))
history_accuracy_train.append(accuracy_train/len(train_loader))
network_learned = valid_loss < valid_loss_min
if epoch == 1 or epoch % 5 == 0 or network_learned:
print(f"Epoch: {epoch}/{EPOCHS}.. ",
f"Training Loss: {running_loss/len(train_loader):.3f}.. ",
f"Validation Loss: {valid_loss/len(valid_loader):.3f}.. ",
f"Training Accuracy: {accuracy_train/len(train_loader):.3f}.. ",
f"Validation Accuracy: {accuracy_val/len(valid_loader):.3f}")
 if network_learned:
valid_loss_min = valid_loss
torch.save(model.state_dict() , 'model_mtl_mnist.pt')
 print('Detected network improvement, saving current model')
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df<data_type_conversions>
train_losses, valid_losses = [], []
history_accuracy_train,history_accuracy_val = [], []
for epoch in range(EPOCHS):
fit(epoch )
def fill_weather_dataset(weather_df):
time_format = "%Y-%m-%d %H:%M:%S"
start_date = datetime.datetime.strptime(weather_df['timestamp'].min() ,time_format)
 end_date = datetime.datetime.strptime(weather_df['timestamp'].max() ,time_format)
 total_hours = int(((end_date - start_date ).total_seconds() + 3600)/ 3600)
 hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format)for x in range(total_hours)]
missing_hours = []
for site_id in range(16):
site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])
 new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])
 new_rows['site_id'] = site_id
weather_df = pd.concat([weather_df,new_rows])
 weather_df = weather_df.reset_index(drop=True)
 weather_df["timestamp"] = pd.to_datetime(weather_df["timestamp"])
 weather_df["day"] = weather_df["timestamp"].dt.day
weather_df["week"] = weather_df["timestamp"].dt.week
weather_df["month"] = weather_df["timestamp"].dt.month
weather_df = weather_df.set_index(['site_id','day','month'])
 air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean() ,columns=["air_temperature"])
 weather_df.update(air_temperature_filler,overwrite=False)
 cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()
cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=["cloud_coverage"])
 weather_df.update(cloud_coverage_filler,overwrite=False)
 due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean() ,columns=["dew_temperature"])
 weather_df.update(due_temperature_filler,overwrite=False)
 sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()
sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])
 weather_df.update(sea_level_filler,overwrite=False)
 wind_direction_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean() ,columns=['wind_direction'])
 weather_df.update(wind_direction_filler,overwrite=False)
 wind_speed_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean() ,columns=['wind_speed'])
 weather_df.update(wind_speed_filler,overwrite=False)
 precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()
precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])
 weather_df.update(precip_depth_filler,overwrite=False)
 weather_df = weather_df.reset_index()
weather_df = weather_df.drop(['day','week','month'],axis=1)
 return weather_df<create_dataframe>
def prediciton() :
model.eval()
test_pred = torch.LongTensor()
for i,data in enumerate(test_loader):
data = Variable(data, requires_grad=True)
 if torch.cuda.is_available() :
data = data.cuda()
output = model(data)
 pred = output.cpu().data.max(1, keepdim=True)[1]
test_pred = torch.cat(( test_pred, pred), dim=0)
 return test_pred
weather_train = fill_weather_dataset(weather_train )<categorify>
test_pred = prediciton()
out_df = pd.DataFrame(np.c_[np.arange(1, len(test_data)+1)[:,None], test_pred.numpy() ],
columns=['ImageId', 'Label'])
 print(out_df )
<compute_test_metric><EOS> 
out_df.to_csv('submission.csv', index=False )