Kaggle competition: Allstate Claims Severity
<SOS> metric: MAE Kaggle data source: allstate-claims-severity<define_variables>
warnings.simplefilter("ignore")
 print(os.listdir(".. /input"))
display_batch_of_images(next(train_batch))<define_variables>
df=pd.read_csv(".. /input/train.csv")
 del df['id']
print(df.shape)
 display(df.head() )
test_dataset = get_test_dataset()
test_dataset = test_dataset.unbatch().batch(20)
 test_batch = iter(test_dataset )<define_variables>
test=pd.read_csv(".. /input/test.csv")
 idx=test['id']
del test['id']
print(test.shape)
 display(test.head() )
display_batch_of_images(next(test_batch))<choose_model_class>
cat_int=['cat12|cat80', 'cat79|cat80', 'cat1|cat12|cat80',
'cat12|cat80|cat81', 'cat12|cat79|cat80', 'cat1|cat80',
'cat103|cat12|cat80', 'cat57|cat80', 'cat12|cat72|cat80',
'cat1|cat103|cat80', 'cat57|cat79|cat80', 'cat1|cat80|cat81',
'cat12|cat57|cat80', 'cat2|cat57|cat80', 'cat10|cat80',
'cat12|cat81', 'cat103|cat12', 'cat1|cat12', 'cat1|cat114|cat80',
'cat10|cat103|cat80', 'cat1|cat111|cat80', 'cat10|cat111|cat80',
'cat12|cat53', 'cat1|cat103', 'cat12|cat53|cat80',
'cat10|cat101|cat80', 'cat12|cat53|cat81', 'cat12|cat79',
'cat1|cat81', 'cat2|cat80|cat81', 'cat81|cat87', 'cat12|cat72',
'cat103|cat111', 'cat1|cat72', 'cat10|cat114|cat81', 'cat4|cat5',
'cat103|cat111|cat80', 'cat10|cat72|cat80', 'cat80|cat81',
'cat103|cat111|cat12', 'cat1|cat2|cat72', 'cat10|cat81|cat87',
'cat2|cat57', 'cat1|cat72|cat79', 'cat79|cat81|cat87',
'cat2|cat44', 'cat114|cat2', 'cat13|cat53', 'cat111|cat13|cat53',
'cat10|cat81']
for each in cat_int:
l=each.split("|")
 if(len(l)==3):
a=l[0]
b=l[1]
c=l[2]
df[a+"_"+b+"_"+c]=df[a]+df[b]+df[c]
test[a+"_"+b+"_"+c]=test[a]+test[b]+test[c]
else:
a=l[0]
b=l[1]
df[a+"_"+b]=df[a]+df[b]
test[a+"_"+b]=test[a]+test[b]
with strategy.scope() :
dn201 = tf.keras.applications.DenseNet201(weights='imagenet', include_top=False, input_shape=[*IMAGE_SIZE, 3])
 dn201.trainable = True
model1 = tf.keras.Sequential([
dn201,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model1.compile(
 optimizer = tf.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, amsgrad=False),
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model1.summary()<choose_model_class>
c=list(df.columns)
 for each in c:
if each[0:3]=='cat':
res=pd.factorize(pd.concat(( df[each],test[each])).values,sort=True)[0]
df[each]=res[0:188318]
test[each]=res[188318:]
with strategy.scope() :
enb7 = efn.EfficientNetB7(weights='noisy-student', include_top=False, input_shape=[*IMAGE_SIZE, 3])
 enb7.trainable = True
model2 = tf.keras.Sequential([
enb7,
tf.keras.layers.GlobalAveragePooling2D() ,
tf.keras.layers.Dense(len(CLASSES), activation='softmax')
 ])
 model2.compile(
 optimizer = tf.keras.optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, amsgrad=False),
loss = 'sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'] 
)
 model2.summary()<randomize_order>
c=list(df.columns)
 c.remove('loss')
 X=df.loc[:,c]
Y=df.loss
X.head()
LR_START = 0.0001
LR_MAX = 0.00005 * strategy.num_replicas_in_sync
LR_MIN = 0.00001
LR_RAMPUP_EPOCHS = 4
LR_SUSTAIN_EPOCHS = 6
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr = np.random.random_sample() * LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True )<choose_model_class>
X['mul_cont2_cont7']=X['cont2']*X['cont7']
X['add_cont2_cont7']=X['cont2']+X['cont7']
X['add_cont2_cont12']=X['cont2']+X['cont12']
X['add_cont2_cont11']=X['cont2']+X['cont11']
X['sub_cont1_cont7']=X['cont1']-X['cont7']
X['add_cont7_cont14']=X['cont7']+X['cont14']
X['div_cont2_cont4']=X['cont2']/X['cont4']
X['add_cont3_cont7']=X['cont3']+X['cont7']
test['mul_cont2_cont7']=test['cont2']*test['cont7']
test['add_cont2_cont7']=test['cont2']+test['cont7']
test['add_cont2_cont12']=test['cont2']+test['cont12']
test['add_cont2_cont11']=test['cont2']+test['cont11']
test['sub_cont1_cont7']=test['cont1']-test['cont7']
test['add_cont7_cont14']=test['cont7']+test['cont14']
test['div_cont2_cont4']=test['cont2']/test['cont4']
test['add_cont3_cont7']=test['cont3']+test['cont7']
es_callback = tf.keras.callbacks.EarlyStopping(min_delta=0, patience=10, verbose=1, mode='auto', restore_best_weights=True )<train_model>
if VALIDATION:
history1 = model1.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=40, validation_data=get_validation_dataset() , callbacks = [lr_callback, es_callback])
 else:
history1 = model1.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=30, callbacks = [lr_callback] )<train_model>
params={'bagging_fraction': 0.5021393514653055,
'feature_fraction': 0.6190345305094423,
'lambda_l1': 0.4932225704923126,
'max_depth': 39,
'min_data_in_leaf': 22,
'min_gain_to_split': 0.2,
'num_leaves': 45,
'silent':True,
'objective':'fair',
'random_state':60,
'n_estimators':10000,
'learning_rate':0.03}
if VALIDATION:
history2 = model2.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=40, validation_data=get_validation_dataset() , callbacks = [lr_callback, es_callback])
 else:
history2 = model2.fit(get_training_dataset() , steps_per_epoch=STEPS_PER_EPOCH, epochs=30, callbacks = [lr_callback] )<predict_on_test>
r=X.nunique()
r.sort_values(ascending=False )
if VALIDATION:
cmdataset = get_validation_dataset(ordered=True)
 images_ds = cmdataset.map(lambda image, label: image)
 labels_ds = cmdataset.map(lambda image, label: label ).unbatch()
cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
cm_probabilities =(model1.predict(images_ds)+model2.predict(images_ds)) /2
cm_predictions = np.argmax(cm_probabilities, axis=-1)
 print("Correct labels: ", cm_correct_labels.shape, cm_correct_labels)
 print("Predicted labels: ", cm_predictions.shape, cm_predictions )<compute_test_metric>
P=np.zeros(( test.shape[0]))
trainALL=[]
valALL=[]
kf = KFold(n_splits=5,random_state=60)
 k=1
for train_index, test_index in kf.split(X):
print("FOLD :",k)
 X_train, X_val = X.loc[train_index,:], X.loc[test_index,:]
Y_train, Y_val = Y[train_index], Y[test_index]
reg=lgb.LGBMRegressor(**params)
 reg.fit(X_train, Y_train,eval_metric='mae',eval_set=[(X_val,Y_val)],verbose=False,early_stopping_rounds=50)
 trainR=mean_absolute_error(np.expm1(Y_train),np.expm1(reg.predict(X_train)))
 valR=mean_absolute_error(np.expm1(Y_val),np.expm1(reg.predict(X_val)))
 print("MAE Train : ",trainR)
 print("MAE Test : ",valR)
 print("****************")
 trainALL.append(trainR)
 valALL.append(valR)
 k+=1
P+=np.expm1(reg.predict(test))
if VALIDATION:
cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))
 score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)) , average='macro')
 display_confusion_matrix(cmat, score, precision, recall)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))<save_to_csv>
print("Train Score: ",np.mean(trainALL)," Std dev:",np.std(trainALL))
print("Val Score: ",np.mean(valALL)," Std dev:",np.std(valALL))
<define_variables><EOS> 
dfx=pd.DataFrame()
dfx['id']=idx
dfx['loss']=P/5
dfx.to_csv("lgb_avgfolds.csv",index=False )