Kaggle competition: Predicting Red Hat Business Value
<SOS> metric: AUC Kaggle data source: predicting-red-hat-business-value<init_hyperparams>
def cleanPeople(people):
people = people.drop(['date'],axis=1)
 people['people_id'] = people['people_id'].apply(lambda x : x.split('_')[1])
 people['people_id'] = pd.to_numeric(people['people_id'] ).astype(int)
 fields = list(people.columns)
 cat_data = fields[1:11]
bool_data = fields[11:]
for data in cat_data:
people[data] = people[data].fillna('type 0')
 people[data] = people[data].apply(lambda x: x.split(' ')[1])
 people[data] = pd.to_numeric(people[data] ).astype(int)
 for data in bool_data:
people[data] = pd.to_numeric(people[data] ).astype(int)
 return people
def cleanAct(data, train=False):
data = data.drop(['date'],axis = 1)
 if train:
data = data.drop(['outcome'],axis=1)
 data['people_id'] = data['people_id'].apply(lambda x : x.split('_')[1])
 data['people_id'] = pd.to_numeric(data['people_id'] ).astype(int)
 data['activity_id'] = data['activity_id'].apply(lambda x: x.split('_')[1])
 data['activity_id'] = pd.to_numeric(data['activity_id'] ).astype(int)
 fields = list(data.columns)
 cat_data = fields[2:13]
for column in cat_data:
data[column] = data[column].fillna('type 0')
 data[column] = data[column].apply(lambda x : x.split(' ')[1])
 data[column] = pd.to_numeric(data[column] ).astype(int)
 return data
def get_lr_callback(batch_size=8):
lr_start = 0.000005
lr_max = 0.00000125 * REPLICAS * batch_size
lr_min = 0.000001
lr_ramp_ep = 5
lr_sus_ep = 0
lr_decay = 0.8
def lrfn(epoch):
if epoch < lr_ramp_ep:
lr =(lr_max - lr_start)/ lr_ramp_ep * epoch + lr_start
elif epoch < lr_ramp_ep + lr_sus_ep:
lr = lr_max
else:
lr =(lr_max - lr_min)* lr_decay**(epoch - lr_ramp_ep - lr_sus_ep)+ lr_min
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)
 return lr_callback<define_variables>
people = pd.read_csv(".. /input/people.csv")
 people = cleanPeople(people)
 act_train = pd.read_csv(".. /input/act_train.csv",parse_dates=['date'])
 act_train_cleaned = cleanAct(act_train,train=True)
 act_test = pd.read_csv(".. /input/act_test.csv",parse_dates=['date'])
 act_test_cleaned = cleanAct(act_test)
 train = act_train_cleaned.merge(people,on='people_id', how='left')
 test = act_test_cleaned.merge(people, on='people_id', how='left' )
VERBOSE = 1
DISPLAY_PLOT = True
hist = []
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 oof_pred = []; oof_tar = []; oof_val = []; oof_names = []; oof_folds = []
preds = np.zeros(( count_data_items(files_test),1))
for fold,(idxT,idxV)in enumerate(skf.split(np.arange(15))):
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print(' 
(IMG_SIZES[fold],EFF_NETS[fold],BATCH_SIZES[fold]*REPLICAS))
files_train = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxT])
 if INC2019[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2+1])
 print('
if INC2018[fold]:
files_train += tf.io.gfile.glob([GCS_PATH2[fold] + '/train%.2i*.tfrec'%x for x in idxT*2])
 print('
np.random.shuffle(files_train); print('
files_valid = tf.io.gfile.glob([GCS_PATH[fold] + '/train%.2i*.tfrec'%x for x in idxV])
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[fold] + '/test*.tfrec')))
 K.clear_session()
with strategy.scope() :
model = build_model(dim=IMG_SIZES[fold],ef=EFF_NETS[fold])
 sv = tf.keras.callbacks.ModelCheckpoint(
 'fold-%i.h5'%fold, monitor='val_loss', verbose=0, save_best_only=True,
save_weights_only=True, mode='min', save_freq='epoch')
 print('Training...')
 history = model.fit(
 get_dataset(files_train, augment=True, shuffle=True, repeat=True,
dim=IMG_SIZES[fold], batch_size = BATCH_SIZES[fold]),
epochs=EPOCHS[fold], callbacks = [sv,get_lr_callback(BATCH_SIZES[fold])],
steps_per_epoch=count_data_items(files_train)/BATCH_SIZES[fold]//REPLICAS,
validation_data=get_dataset(files_valid,augment=False,shuffle=False,
repeat=False,dim=IMG_SIZES[fold]),
verbose=VERBOSE 
)
 hist.append(history.history)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF with TTA...')
 ds_valid = get_dataset(files_valid,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_valid = count_data_items(files_valid); STEPS = TTA * ct_valid/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_valid,steps=STEPS,verbose=VERBOSE)[:TTA*ct_valid,]
oof_pred.append(np.mean(pred.reshape(( ct_valid,TTA),order='F'),axis=1))
ds_valid = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=True, return_image_names=True)
 oof_tar.append(np.array([target.numpy() for img, target in iter(ds_valid.unbatch())]))
oof_folds.append(np.ones_like(oof_tar[-1],dtype='int8')*fold)
 ds = get_dataset(files_valid, augment=False, repeat=False, dim=IMG_SIZES[fold],
labeled=False, return_image_names=True)
 oof_names.append(np.array([img_name.numpy().decode("utf-8")for img, img_name in iter(ds.unbatch())]))
print('Predicting Test with TTA...')
 ds_test = get_dataset(files_test,labeled=False,return_image_names=False,augment=True,
repeat=True,shuffle=False,dim=IMG_SIZES[fold],batch_size=BATCH_SIZES[fold]*4)
 ct_test = count_data_items(files_test); STEPS = TTA * ct_test/BATCH_SIZES[fold]/4/REPLICAS
pred = model.predict(ds_test,steps=STEPS,verbose=VERBOSE)[:TTA*ct_test,]
preds[:,0] += np.mean(pred.reshape(( ct_test,TTA),order='F'),axis=1)* WGTS[fold]
auc = roc_auc_score(oof_tar[-1],oof_pred[-1])
 oof_val.append(np.max(history.history['val_auc']))
print('
if DISPLAY_PLOT:
plt.figure(figsize=(15,5))
plt.plot(np.arange(EPOCHS[fold]),history.history['auc'],'-o',label='Train AUC',color='
plt.plot(np.arange(EPOCHS[fold]),history.history['val_auc'],'-o',label='Val AUC',color='
x = np.argmax(history.history['val_auc']); y = np.max(history.history['val_auc'])
 xdist = plt.xlim() [1] - plt.xlim() [0]; ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
%.2f'%y,size=14)
 plt.ylabel('AUC',size=14); plt.xlabel('Epoch',size=14)
 plt.legend(loc=2)
 plt2 = plt.gca().twinx()
plt2.plot(np.arange(EPOCHS[fold]),history.history['loss'],'-o',label='Train Loss',color='
plt2.plot(np.arange(EPOCHS[fold]),history.history['val_loss'],'-o',label='Val Loss',color='
x = np.argmin(history.history['val_loss']); y = np.min(history.history['val_loss'])
 ydist = plt.ylim() [1] - plt.ylim() [0]
plt.scatter(x,y,s=200,color='
plt.ylabel('Loss',size=14)
 plt.title('FOLD %i - Image Size %i, EfficientNet B%i, inc2019=%i, inc2018=%i'% 
(fold+1,IMG_SIZES[fold],EFF_NETS[fold],INC2019[fold],INC2018[fold]),size=18)
 plt.legend(loc=3)
 plt.show()<save_model>
train = pd.get_dummies(train,columns=['activity_category'],sparse=True,drop_first=True)
 test = pd.get_dummies(test,columns=['activity_category'],sparse=True,drop_first=True )
np.save("history.npy", hist )<save_to_csv>
train_mask, valid_mask = list(LabelKFold(train['people_id'], n_folds=10)) [0]
x_test = test.drop(['people_id','activity_id'],axis=1)
 y = act_train['outcome']
train = train.drop(['people_id', 'activity_id'], axis=1)
 kklo=x_train = np.array(train)[train_mask]
y_train = np.array(y)[train_mask]
x_valid = np.array(train)[valid_mask]
y_valid = np.array(y)[valid_mask]
params = {}
params['objective'] = 'binary:logistic'
params['eval_metric'] = 'auc'
params['eta'] = 0.1
params['max_depth'] = 10
params['subsample'] = 0.9
params['colsample_bytree'] = 0.9
d_train = xgb.DMatrix(x_train, label=y_train, missing=np.nan)
 d_valid = xgb.DMatrix(x_valid, label=y_valid, missing=np.nan)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
clf = xgb.train(params, d_train, 600, watchlist, early_stopping_rounds=40)
 clf.save_model('mymodel.model' )
oof = np.concatenate(oof_pred); true = np.concatenate(oof_tar);
names = np.concatenate(oof_names); folds = np.concatenate(oof_folds)
 auc = roc_auc_score(true,oof)
 print('Overall OOF AUC with TTA = %.3f'%auc)
 df_oof = pd.DataFrame(dict(
 image_name = names, target=true, pred = oof, fold=folds))
df_oof.to_csv('oof.csv',index=False)
 df_oof.head()<create_dataframe>
clf = xgb.train(params, d_train, 800, watchlist, xgb_model ='mymodel.model', early_stopping_rounds=40 )
<save_to_csv><EOS> 
p_test = clf.predict(xgb.DMatrix(np.array(x_test)))
 sub = pd.DataFrame()
sub['activity_id'] = act_test['activity_id']
sub['outcome'] = p_test
sub.to_csv('submission.csv', index=False )