Kaggle competition: Predicting Red Hat Business Value
<SOS> metric: AUC Kaggle data source: predicting-red-hat-business-value<normalization>
act_train = pd.read_csv('.. /input/act_train.csv')
 act_test = pd.read_csv('.. /input/act_test.csv')
 people = pd.read_csv('.. /input/people.csv')
 people.sample(10 )
plot_transform(7 )<define_variables>
def process_dates(data,min_date):
min_date
data=data.apply(lambda x:(datetime.datetime.strptime(x,"%Y-%m-%d")
 -datetime.datetime.strptime(min_date,"%Y-%m-%d")).days)
 data
return data
train_dataset =(load_dataset(train_filenames, labeled=True)
.map(data_augment, num_parallel_calls=AUTO)
.shuffle(SEED)
.batch(BATCH_SIZE,drop_remainder=True)
.repeat() 
.prefetch(AUTO))
valid_dataset =(load_dataset(valid_filenames, labeled=True)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO))<compute_test_metric>
test_ids = act_test['activity_id']
def preprocess_acts(data,min_date, train_set=True):
dates=data['date']
dates=process_dates(dates,min_date)
 data = data.drop(['date', 'activity_id'], axis=1)
 if(train_set):
data = data.drop(['outcome'], axis=1)
 data['people_id'] = data['people_id'].apply(lambda x: x.split('_')[1])
 data['people_id'] = pd.to_numeric(data['people_id'] ).astype(int)
 columns = list(data.columns)
 for col in columns[1:]:
data[col] = data[col].fillna('type 0')
 data[col] = data[col].apply(lambda x: x.split(' ')[1])
 data[col] = pd.to_numeric(data[col] ).astype(int)
 data['dates']=dates
return data
def preprocess_people(data,min_date):
dates=data['date']
dates=process_dates(dates,min_date)
 data = data.drop(['date'], axis=1)
 data['people_id'] = data['people_id'].apply(lambda x: x.split('_')[1])
 data['people_id'] = pd.to_numeric(data['people_id'] ).astype(int)
 columns = list(data.columns)
 bools = columns[11:]
strings = columns[1:11]
for col in bools:
data[col] = pd.to_numeric(data[col] ).astype(int)
 for col in strings:
data[col] = data[col].fillna('type 0')
 data[col] = data[col].apply(lambda x: x.split(' ')[1])
 data[col] = pd.to_numeric(data[col] ).astype(int)
 data['dates']=dates
return data
def focal_loss(alpha=0.25,gamma=2.0):
def focal_crossentropy(y_true, y_pred):
bce = K.binary_crossentropy(y_true, y_pred)
 y_pred = K.clip(y_pred, K.epsilon() , 1.- K.epsilon())
 p_t =(y_true*y_pred)+(( 1-y_true)*(1-y_pred))
alpha_factor = 1
modulating_factor = 1
alpha_factor = y_true*alpha +(( 1-alpha)*(1-y_true))
modulating_factor = K.pow(( 1-p_t), gamma)
 return K.mean(alpha_factor*modulating_factor*bce, axis=-1)
 return focal_crossentropy<choose_model_class>
min_date=pd.concat([people['date'],act_train['date'],act_test['date']] ).min()
min_date
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB7(input_shape=(*SIZE, 3),weights='noisy-student',pooling='avg',include_top=False),
Dense(1, activation='sigmoid')
 ])
 model.compile(
 optimizer='adam',
loss = focal_loss() ,
metrics=['accuracy',tf.keras.metrics.AUC(name='auc')] )<train_model>
min_date=pd.concat([people['date'],act_train['date'],act_test['date']] ).min()
peeps = preprocess_people(people,min_date)
 actions_train = preprocess_acts(act_train,min_date,train_set=True)
 actions_test = preprocess_acts(act_test,min_date,train_set=False)
 print(peeps.columns)
 print(actions_train.columns)
 peeps.sample(10 )
def train() :
STEPS_PER_EPOCH = count_data_items(train_filenames)// BATCH_SIZE
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=STEPS_PER_EPOCH,
workers=8,
use_multiprocessing=True,
validation_data=valid_dataset)
 string = 'Train acc:{:.4f} Train loss:{:.4f} AUC: {:.4f}, Val acc:{:.4f} Val loss:{:.4f} Val AUC: {:.4f}'.format(\
model.history.history['accuracy'][-1],model.history.history['loss'][-1],\
model.history.history['auc'][-1],\
model.history.history['val_accuracy'][-1],model.history.history['val_loss'][-1],\
model.history.history['val_auc'][-1])
 return string 
<train_model>
features = actions_train.merge(peeps, how='left', on='people_id')
 features=features.drop(['people_id'],axis=1)
 labels = act_train['outcome']
test = actions_test.merge(peeps, how='left', on='people_id')
 test=test.drop(['people_id'],axis=1)
 features.sample(10 )
train()<load_from_csv>
num_test = 0.10
X_train, X_test, y_train, y_test = train_test_split(features, labels, test_size=num_test, random_state=23)
 clf = RandomForestClassifier()
clf.fit(X_train, y_train )
num_test_images = count_data_items(test_filenames)
 submission_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 for i in range(TTA):
test_dataset =(load_dataset(test_filenames, labeled=False,ordered=True)
.map(data_augment, num_parallel_calls=AUTO)
.batch(BATCH_SIZE))
test_dataset_images = test_dataset.map(lambda image, image_name: image)
 test_dataset_image_name = test_dataset.map(lambda image, image_name: image_name ).unbatch()
test_ids = next(iter(test_dataset_image_name.batch(num_test_images)) ).numpy().astype('U')
 test_pred = model.predict(test_dataset_images, verbose=1)
 pred_df = pd.DataFrame({'image_name': test_ids, 'target': np.concatenate(test_pred)})
 temp = submission_df.copy()
del temp['target']
submission_df['target'] += temp.merge(pred_df,on="image_name")['target']/TTA<save_to_csv>
proba = clf.predict_proba(X_test)
 preds = proba[:,1]
score = roc_auc_score(y_test, preds)
 print("Area under ROC {0}".format(score))
submission_df.to_csv('submission.csv', index=False)
 pd.Series(np.round(submission_df['target'].values)).value_counts()<set_options>
test_proba = clf.predict_proba(test)
 test_preds = test_proba[:,1]
test_res=clf.predict(test)
 output = pd.DataFrame({ 'activity_id' : test_ids, 'outcome': test_preds })
 output1 = pd.DataFrame({ 'activity_id' : test_ids, 'outcome': test_res })
 output.head()
<set_options><EOS> 
output.to_csv('redhat.csv', index = False)
 output1.to_csv('redhat_noprpba.csv', index = False )