Kaggle competition: WiDS Datathon 2019
all_preds = pd.DataFrame()
base_test = get_base_test()
main_time = time.time()
for PREDICT_DAY in range(1,29):
print('Predict | Day:', PREDICT_DAY)
 start_time = time.time()
grid_df = base_test.copy()
grid_df = pd.concat([grid_df, df_parallelize_run(make_lag_roll, ROLS_SPLIT)], axis=1)
 for store_id in STORES_IDS:
model_path = 'lgb_model_'+store_id+'_v'+str(VER)+'.bin'
if USE_AUX:
model_path = AUX_MODELS + model_path
estimator = pickle.load(open(model_path, 'rb'))
day_mask = base_test['d']==(END_TRAIN+PREDICT_DAY)
 store_mask = base_test['store_id']==store_id
mask =(day_mask)&(store_mask)
 base_test[TARGET][mask] = estimator.predict(grid_df[mask][MODEL_FEATURES])
 temp_df = base_test[day_mask][['id',TARGET]]
temp_df.columns = ['id','F'+str(PREDICT_DAY)]
if 'id' in list(all_preds):
all_preds = all_preds.merge(temp_df, on=['id'], how='left')
 else:
all_preds = temp_df.copy()
print('
' %0.2f min total |' %(( time.time() - main_time)/ 60),
' %0.2f day sales |' %(temp_df['F'+str(PREDICT_DAY)].sum()))
del temp_df
all_preds = all_preds.reset_index(drop=True)
 all_preds<save_to_csv>
%matplotlib inline
submission = pd.read_csv(ORIGINAL+'sample_submission.csv')[['id']]
submission = submission.merge(all_preds, on=['id'], how='left' ).fillna(0)
 submission.to_csv('submission_v'+str(VER)+'.csv', index=False )<set_options>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )
warnings.filterwarnings("ignore")
<load_from_csv>
df = pd.read_csv('.. /input/widsdatathon2019/traininglabels.csv')
 df.head()
train = pd.read_csv('.. /input/higgs-boson/training.zip')
 test = pd.read_csv('.. /input/higgs-boson/test.zip')
 print(train.shape,test.shape )<drop_column>
class OilDataset(Dataset):
def __init__(self,csv,transform):
self.data = pd.read_csv(csv)
 self.transform = transform
self.label = torch.eye(2)[self.data['has_oilpalm']]
def __len__(self):
return len(self.data)
 def __getitem__(self,idx):
image_path = os.path.join('.. /input/widsdatathon2019/train_images/train_images/'+self.data.loc[idx,'image_id'])
 image = Image.open(image_path)
 image = self.transform(image)
 label = torch.tensor(self.data.loc[idx,'has_oilpalm'])
 return {'image':image,'labels':label}
train = train.drop(['Weight'], axis=1 )<categorify>
k = Image.open('.. /input/widsdatathon2019/train_images/train_images/img_000012017.jpg')
 k.show()
enc = LabelEncoder()
train['Label'] = enc.fit_transform(train['Label'])
 train.head()<prepare_x_and_y>
simple_transform = transforms.Compose([transforms.Resize(( 224,224)) ,transforms.RandomHorizontalFlip() ,transforms.RandomRotation(45),
transforms.ToTensor()
,transforms.Normalize([0.486,0.456,0.406],[0.229,0.225,0.224])] )
y = train["Label"]
X = train
X_test = test<drop_column>
train_dataset = OilDataset('.. /input/widsdatathon2019/traininglabels.csv',simple_transform )
X.set_index(['EventId'],inplace = True)
 X_test.set_index(['EventId'],inplace = True)
 X = X.drop(['Label'], axis=1)
 X.head()<normalization>
data_size = len(train_dataset)
 indics = list(range(data_size))
X = normalize(X)
 X_test = normalize(X_test )<choose_model_class>
np.random.shuffle(indics )
<split>
validation_split = 0.1
split = int(np.round(validation_split*data_size))
train_indices,validation_indices = indics[split:],indics[:split]
kf = KFold(n_splits=5, random_state=2020, shuffle=True)
 for train_index, val_index in kf.split(X):
print("TRAIN:", train_index, "TEST:", val_index)
 X_train, X_val = X[train_index], X[val_index]
y_train, y_val = y[train_index], y[val_index]<train_model>
train_sampler = SubsetRandomSampler(train_indices)
 valid_sampler = SubsetRandomSampler(validation_indices )
<categorify>
train_loader = DataLoader(train_dataset,batch_size=32,sampler=train_sampler)
 valid_loader = DataLoader(train_dataset,batch_size=32,sampler = valid_sampler )
X_train = X_train.reshape(-1, 1, 30)
 X_val = X_val.reshape(-1, 1, 30)
 y_train = y_train.values
y_train = y_train.reshape(-1, 1,)
 y_val = y_val.values
y_val = y_val.reshape(-1, 1, )<choose_model_class>
model = resnet50(pretrained=False)
 model.load_state_dict(torch.load('.. /input/resnet50/resnet50.pth'))
for params in model.parameters() :
params.require_grad = False
model.fc = nn.Sequential(
 nn.Linear(2048,1024),
nn.ReLU() ,
nn.Linear(1024,5)
)
 fc_parameter = model.fc.parameters()
for params in fc_parameter:
params.require_grad = True
model = model.to(device )
input_layer = Input(shape=(1,30))
main_rnn_layer = LSTM(64, return_sequences=True, recurrent_dropout=0.2 )(input_layer)
 rnn = LSTM(32 )(main_rnn_layer)
 dense = Dense(128 )(rnn)
 dropout_c = Dropout(0.3 )(dense)
 classes = Dense(1, activation= LeakyReLU(alpha=0.1),name="class" )(dropout_c)
 model = Model(input_layer, classes)
 callbacks = [ReduceLROnPlateau(monitor='val_loss', patience=4, verbose=1, factor=0.6),
EarlyStopping(monitor='val_loss', patience=20),
ModelCheckpoint(filepath='best_model.h5', monitor='val_loss', save_best_only=True)]
model.compile(loss=[tf.keras.losses.MeanSquaredLogarithmicError() ,tf.keras.losses.MeanSquaredLogarithmicError() ], optimizer="adam")
 model.summary()
history = model.fit(X_train, y_train,
epochs = 250,
batch_size = 16,
validation_data=(X_val, y_val),
callbacks=callbacks)
<predict_on_test>
criteria = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters() ,lr=0.001 )
model.load_weights("best_model.h5")
 test = X_test
test = test.reshape(-1, 1,30)
 predictions = model.predict(test )<load_from_csv>
def fit(epochs,model,optimizer,criteria):
for epoch in range(epochs+1):
training_loss = 0.0
validation_loss = 0.0
correct = 0
total = 0
print('{}/{} Epoch'.format(epoch+1,epochs))
model.train()
for batch_idx,d in enumerate(train_loader):
data = d['image'].cuda()
target = d['labels'].cuda()
optimizer.zero_grad()
output = model(data)
 loss = criteria(output,target)
 loss.backward()
optimizer.step()
training_loss = training_loss +(( 1/(batch_idx+1)*(loss.data-training_loss)))
 if batch_idx%10==0:
print('Loss is of {}'.format(training_loss))
pred = output.data.max(1,keepdim = True)[1]
correct+=np.sum(np.squeeze(pred.eq(target.data.view_as(pred)) ).cpu().numpy())
 total +=data.size(0)
 if batch_idx%40==0:
print('Accuracy for batch {} on Training {}'.format(batch_idx,(100*(correct/total))))
model.eval()
for batch_idx,d in enumerate(valid_loader):
data = d['image'].cuda()
target = d['labels'].cuda()
output = model(data)
 loss = criteria(output,target)
 validation_loss = validation_loss+(((1/(batch_idx+1)) *(loss.data-validation_loss)))
 pred = output.data.max(1,keepdim =True)[1]
correct+=np.sum(np.squeeze(pred.eq(target.data.view_as(pred)) ).cpu().numpy())
 total += data.size(0)
 if batch_idx%40==0:
print('Validation Accuracy {}'.format(( 100*correct/total)))
 return model
sub = pd.read_csv('.. /input/higgs-boson/random_submission.zip' )<prepare_output>
fit(12,model,optimizer,criteria )
pred = np.where(predictions > 0.5, 1, 0)
 pred<create_dataframe>
class prediction(Dataset):
def __init__(self,csv,transform):
self.test_data = pd.read_csv(csv)
 self.transform = transform
def __len__(self):
return len(self.test_data)
 def __getitem__(self,idx):
try:
image_path = os.path.join('.. /input/widsdatathon2019/leaderboard_test_data/leaderboard_test_data/',self.test_data.loc[idx]['image_id'])
 image = Image.open(image_path)
 except:
image_path = os.path.join('.. /input/widsdatathon2019/leaderboard_holdout_data/leaderboard_holdout_data/',self.test_data.loc[idx]['image_id'])
 image = Image.open(image_path)
 image = self.transform(image)
 name = self.test_data.loc[idx]['image_id']
return {'images':image,'names':name}
test_predict = pd.DataFrame({"EventId":sub['EventId'],"RankOrder":sub['RankOrder'],"Class":test_predict})
 test_predict<prepare_output>
predict_dataset = prediction('.. /input/widsdatathon2019/SampleSubmission.csv',simple_transform )
test_predict = test_predict.replace(1,'s')
 test_predict = test_predict.replace(0,'b')
 test_predict<feature_engineering>
prediction_loader = DataLoader(predict_dataset )
test_predict['RankOrder'] = test_predict['Class'].argsort().argsort() + 1<save_to_csv>
gc.collect()
test_predict.to_csv("submission.csv",index=False )<define_variables>
predict = []
model.eval()
for i, d in enumerate(prediction_loader):
data = d['images'].cuda()
output = model(data)
 output1 = output.cpu().detach().numpy()
predict.append(output1[0] )
DEBUG = False<define_variables>
submission = pd.read_csv('.. /input/widsdatathon2019/SampleSubmission.csv')
 submission['has_oilpalm'] = np.argmax(predict,1 )
<import_modules><EOS> 
submission.to_csv('sample_submission.csv',index = False )