Kaggle competition: Categorical Feature Encoding Challenge
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = 0.0
for batch in range(batch_size):
t, p = A[batch], B[batch]
true = np.sum(t)
 pred = np.sum(p)
 if true == 0:
metric +=(pred == 0)
 continue
intersection = np.sum(t * p)
 union = true + pred - intersection
iou = intersection / union
iou = np.floor(max(0,(iou - 0.45)*20)) / 10
metric += iou
metric /= batch_size
return metric
def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred > 0.5], tf.float64 )<compute_test_metric>
train = pd.read_csv('/kaggle/input/cat-in-the-dat/train.csv')
 test = pd.read_csv('/kaggle/input/cat-in-the-dat/test.csv' )
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
 y_pred = K.cast(y_pred, 'float32')
 y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
 intersection = y_true_f * y_pred_f
score = 2.* K.sum(intersection)/(K.sum(y_true_f)+ K.sum(y_pred_f))
return score
def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred)
 def bce_logdice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)- K.log(1.- dice_loss(y_true, y_pred))<choose_model_class>
train.nunique()
class SnapshotCallbackBuilder:
def __init__(self, nb_epochs, nb_snapshots, init_lr=0.1):
self.T = nb_epochs
self.M = nb_snapshots
self.alpha_zero = init_lr
def get_callbacks(self, model_prefix='Model'):
callback_list = [
callbacks.ModelCheckpoint("./keras.model",monitor='val_my_iou_metric',
mode = 'max', save_best_only=True, verbose=1),
swa,
callbacks.LearningRateScheduler(schedule=self._cosine_anneal_schedule)
 ]
return callback_list
def _cosine_anneal_schedule(self, t):
cos_inner = np.pi *(t %(self.T // self.M))
cos_inner /= self.T // self.M
cos_out = np.cos(cos_inner)+ 1
return float(self.alpha_zero / 2 * cos_out )<define_search_model>
train.isna().sum()
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 x = BatchNormalization()(x)
 if activation == True:
x = LeakyReLU(alpha=0.1 )(x)
 return x
def residual_block(blockInput, num_filters=16):
x = LeakyReLU(alpha=0.1 )(blockInput)
 x = BatchNormalization()(x)
 blockInput = BatchNormalization()(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 return x<choose_model_class>
bin_cols = [col for col in train.columns.values if col.startswith('bin')]
nom_cols = [col for col in train.columns.values if col.startswith('nom')]
ord_cols = [col for col in train.columns.values if col.startswith('ord')]
def UEfficientNet(input_shape=(None, None, 3),dropout_rate=0.1):
backbone = EfficientNetB4(weights='imagenet',
include_top=False,
input_shape=input_shape)
 input = backbone.input
start_neurons = 8
conv4 = backbone.layers[342].output
conv4 = LeakyReLU(alpha=0.1 )(conv4)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(dropout_rate )(pool4)
 convm = Conv2D(start_neurons * 32,(3, 3), activation=None, padding="same",name='conv_middle' )(pool4)
 convm = residual_block(convm,start_neurons * 32)
 convm = residual_block(convm,start_neurons * 32)
 convm = LeakyReLU(alpha=0.1 )(convm)
 deconv4 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(convm)
 deconv4_up1 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(deconv4)
 deconv4_up2 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(deconv4_up1)
 deconv4_up3 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(deconv4_up2)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(dropout_rate )(uconv4)
 uconv4 = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 16)
 uconv4 = LeakyReLU(alpha=0.1 )(uconv4)
 deconv3 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(uconv4)
 deconv3_up1 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(deconv3)
 deconv3_up2 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(deconv3_up1)
 conv3 = backbone.layers[154].output
uconv3 = concatenate([deconv3,deconv4_up1, conv3])
 uconv3 = Dropout(dropout_rate )(uconv3)
 uconv3 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 8)
 uconv3 = LeakyReLU(alpha=0.1 )(uconv3)
 deconv2 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 deconv2_up1 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="same" )(deconv2)
 conv2 = backbone.layers[92].output
uconv2 = concatenate([deconv2,deconv3_up1,deconv4_up2, conv2])
 uconv2 = Dropout(0.1 )(uconv2)
 uconv2 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 4)
 uconv2 = LeakyReLU(alpha=0.1 )(uconv2)
 deconv1 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv2)
 conv1 = backbone.layers[30].output
uconv1 = concatenate([deconv1,deconv2_up1,deconv3_up2,deconv4_up3, conv1])
 uconv1 = Dropout(0.1 )(uconv1)
 uconv1 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 2)
 uconv1 = LeakyReLU(alpha=0.1 )(uconv1)
 uconv0 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="same" )(uconv1)
 uconv0 = Dropout(0.1 )(uconv0)
 uconv0 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv0)
 uconv0 = residual_block(uconv0,start_neurons * 1)
 uconv0 = LeakyReLU(alpha=0.1 )(uconv0)
 uconv0 = Dropout(dropout_rate/2 )(uconv0)
 output_layer = Conv2D(1,(1,1), padding="same", activation="sigmoid" )(uconv0)
 model = Model(input, output_layer)
 model.name = 'u-xception'
return model<choose_model_class>
def ord_to_num(df, col):
keys=np.sort(df[col].unique())
 values=np.arange(len(keys))
map = dict(zip(keys, values))
df[col] = df[col].replace(map )
K.clear_session()
img_size = 256
model = UEfficientNet(input_shape=(img_size,img_size,3),dropout_rate=0.5 )<train_model>
ord_to_num(test, 'ord_3')
 ord_to_num(train, 'ord_3' )
class SWA(keras.callbacks.Callback):
def __init__(self, filepath, swa_epoch):
super(SWA, self ).__init__()
self.filepath = filepath
self.swa_epoch = swa_epoch
def on_train_begin(self, logs=None):
self.nb_epoch = self.params['epochs']
print('Stochastic weight averaging selected for last {} epochs.' 
.format(self.nb_epoch - self.swa_epoch))
def on_epoch_end(self, epoch, logs=None):
if epoch == self.swa_epoch:
self.swa_weights = self.model.get_weights()
elif epoch > self.swa_epoch:
for i in range(len(self.swa_weights)) :
self.swa_weights[i] =(self.swa_weights[i] * 
(epoch - self.swa_epoch)+ self.model.get_weights() [i])/(( epoch - self.swa_epoch)+ 1)
 else:
pass
def on_train_end(self, logs=None):
self.model.set_weights(self.swa_weights)
 print('Final model parameters set to stochastic weight average.')
 self.model.save_weights(self.filepath)
 print('Final stochastic averaged weights saved to file.' )<choose_model_class>
ord_to_num(train, 'ord_4')
 ord_to_num(test,'ord_4' )
model.compile(loss=bce_dice_loss, optimizer='adam', metrics=[my_iou_metric] )<choose_model_class>
ord_to_num(train, 'ord_5')
 ord_to_num(test,'ord_5' )
epochs = 70
snapshot = SnapshotCallbackBuilder(nb_epochs=epochs,nb_snapshots=1,init_lr=1e-3)
 batch_size = 16
swa = SWA('./keras_swa.model',67)
 valid_im_path,valid_mask_path = './keras_im_val','./keras_mask_val'
training_generator = DataGenerator(augmentations=AUGMENTATIONS_TRAIN,img_size=img_size)
 validation_generator = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST,
img_size=img_size)
 history = model.fit_generator(generator=training_generator,
validation_data=validation_generator,
use_multiprocessing=False,
epochs=epochs,verbose=2,
callbacks=snapshot.get_callbacks() )<load_pretrained>
values_ord_1=[3,4,0,1,2]
try:
print('using swa weight model')
 model.load_weights('./keras_swa.model')
 except Exception as e:
print(e)
 model.load_weights('./keras.model' )<predict_on_test>
map_ord_1 = dict(zip(keys_ord_1, values_ord_1))
map_ord_1
def predict_result(model,validation_generator,img_size):
preds_test1 = model.predict_generator(validation_generator ).reshape(-1, img_size, img_size)
 return preds_test1<choose_model_class>
train['ord_1'] = train['ord_1'].replace(map_ord_1)
 test['ord_1'] = test['ord_1'].replace(map_ord_1 )
validation_generator = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST,
img_size=img_size,shuffle=False)
 AUGMENTATIONS_TEST_FLIPPED = Compose([
HorizontalFlip(p=1),
ToFloat(max_value=1)
 ],p=1)
 validation_generator_flipped = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST_FLIPPED,
img_size=img_size,shuffle=False)
 preds_valid_orig = predict_result(model,validation_generator,img_size)
 preds_valid_flipped = predict_result(model,validation_generator_flipped,img_size)
 preds_valid_flipped = np.array([np.fliplr(x)for x in preds_valid_flipped])
 preds_valid = 0.5*preds_valid_orig + 0.5*preds_valid_flipped<train_model>
values_ord_2=[1,3,5,4,0,2]
valid_fn = glob.glob('./keras_mask_val/*')
 y_valid_ori = np.array([cv2.resize(np.array(Image.open(fn)) ,(img_size,img_size)) for fn in valid_fn])
 assert y_valid_ori.shape == preds_valid.shape<define_variables>
map_ord_2 = dict(zip(keys_ord_2, values_ord_2))
map_ord_2
threshold_best = 0.5
max_images = 64
grid_width = 16
grid_height = int(max_images / grid_width)
 fig, axs = plt.subplots(grid_height, grid_width, figsize=(grid_width, grid_height))
validation_generator = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST,
img_size=img_size,batch_size=64,shuffle=False)
 images,masks = validation_generator.__getitem__(0)
 for i,(im, mask)in enumerate(zip(images,masks)) :
pred = preds_valid[i]
ax = axs[int(i / grid_width), i % grid_width]
ax.imshow(im[...,0], cmap="bone")
 ax.imshow(mask.squeeze() , alpha=0.5, cmap="Reds")
 ax.imshow(np.array(np.round(pred > threshold_best), dtype=np.float32), alpha=0.5, cmap="Greens")
 ax.axis('off')
 plt.suptitle("Green:Prediction , Red: Pneumothorax." )<compute_train_metric>
train['ord_2'] = train['ord_2'].replace(map_ord_2)
 test['ord_2'] = test['ord_2'].replace(map_ord_2 )
thresholds = np.linspace(0.2, 0.9, 31)
 ious = np.array([iou_metric_batch(y_valid_ori, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)] )<define_variables>
train[ord_cols].nunique()
max_images = 64
grid_width = 16
grid_height = int(max_images / grid_width)
 fig, axs = plt.subplots(grid_height, grid_width, figsize=(grid_width, grid_height))
validation_generator = DataGenerator(train_im_path = valid_im_path ,
train_mask_path=valid_mask_path,augmentations=AUGMENTATIONS_TEST,
img_size=img_size,batch_size=64,shuffle=False)
 images,masks = validation_generator.__getitem__(0)
 for i,(im, mask)in enumerate(zip(images,masks)) :
pred = preds_valid[i]
ax = axs[int(i / grid_width), i % grid_width]
ax.imshow(im[...,0], cmap="bone")
 ax.imshow(mask.squeeze() , alpha=0.5, cmap="Reds")
 ax.imshow(np.array(np.round(pred > threshold_best), dtype=np.float32), alpha=0.5, cmap="Greens")
 ax.axis('off')
 plt.suptitle("Green:Prediction , Red: Pneumothorax." )<predict_on_test>
train['ord_4_band'] = pd.qcut(train['ord_4'], 6)
 bands=train.ord_4_band.unique()
keys_bands=np.sort(bands)
 values_bands=np.arange(len(keys_bands))
map_bands = dict(zip(keys_bands, values_bands))
test_fn = glob.glob('./test/*')
 x_test = [cv2.resize(np.array(Image.open(fn)) ,(img_size,img_size)) for fn in test_fn]
x_test = np.array(x_test)
 x_test = np.array([np.repeat(im[...,None],3,2)for im in x_test])
 print(x_test.shape)
 preds_test_orig = model.predict(x_test,batch_size=batch_size)
 x_test = np.array([np.fliplr(x)for x in x_test])
 preds_test_flipped = model.predict(x_test,batch_size=batch_size)
 preds_test_flipped = np.array([np.fliplr(x)for x in preds_test_flipped])
 preds_test = 0.5*preds_test_orig + 0.5*preds_test_flipped 
<categorify>
train['ord_4_band'] = train['ord_4_band'].replace(map_bands)
 test['ord_4_band']=pd.cut(test.ord_4,pd.IntervalIndex(keys_bands))
sys.path.insert(0, '.. /input/siim-acr-pneumothorax-segmentation')
 rles = []
i,max_img = 1,10
plt.figure(figsize=(16,4))
for p in tqdm_notebook(preds_test):
p = p.squeeze()
im = cv2.resize(p,(1024,1024))
im = im > threshold_best
if im.sum() <1024*2:
im[:] = 0
im =(im.T*255 ).astype(np.uint8)
 rles.append(mask2rle(im, 1024, 1024))
i += 1
if i<max_img:
plt.subplot(1,max_img,i)
 plt.imshow(im)
 plt.axis('off' )<prepare_output>
test['ord_4_band'] = test['ord_4_band'].replace(map_bands)
 test.ord_4_band.head()
ids = [o.split('/')[-1][:-4] for o in test_fn]
sub_df = pd.DataFrame({'ImageId': ids, 'EncodedPixels': rles})
 sub_df.loc[sub_df.EncodedPixels=='', 'EncodedPixels'] = '-1'
sub_df.head()<save_to_csv>
train['ord_5_band'] = pd.qcut(train['ord_5'], 6)
 bands=train.ord_5_band.unique()
keys_bands=np.sort(bands)
 values_bands=np.arange(len(keys_bands))
map_bands = dict(zip(keys_bands, values_bands))
sub_df.to_csv('submission.csv', index=False )<set_options>
train['ord_5_band'] = train['ord_5_band'].replace(map_bands)
 test['ord_5_band']=pd.cut(test.ord_5,pd.IntervalIndex(keys_bands))
%reload_ext autoreload
%autoreload 2
%matplotlib inline
sys.path.insert(0, '.. /input/siim-acr-pneumothorax-segmentation')
<load_pretrained>
test['ord_5_band'] = test['ord_5_band'].replace(map_bands)
 test.ord_5_band.head()
sz = 256
noise_th = 75.0*255*(sz/128.0)**2
nfolds = 4
MASKS = '.. /input/siimacr-pneumothorax-segmentation-data-256/masks'
DATA = '.. /input/hypercolumns-pneumothorax-fastai-0-819-lb'
preds0, items = [], []
for fold in range(nfolds):
preds0.append(torch.load(os.path.join(DATA,'preds_fold'+str(fold)+'.pt')))
 items.append(np.load(os.path.join(DATA,'items_fold'+str(fold)+'.npy'),
allow_pickle=True))
preds0 = torch.cat(preds0)
 items = np.concatenate(items)
 ys = []
for item in items:
img = cv2.imread(str(item ).replace('train', 'masks'),0)
 img = torch.Tensor(img ).byte().unsqueeze(0)
 img =(img > 127)
 ys.append(img)
 ys = torch.cat(ys )<feature_engineering>
train[nom_cols].nunique()
def dice_overall(preds, targs):
n = preds.shape[0]
preds = preds.view(n, -1)
 targs = targs.view(n, -1)
 intersect =(preds * targs ).float().sum(-1)
 union =(preds+targs ).float().sum(-1)
 u0 = union==0
intersect[u0] = 1
union[u0] = 2
return(2.* intersect / union )<feature_engineering>
test[nom_cols].nunique()
TEST = '.. /input/siimacr-pneumothorax-segmentation-data-256/test/'
SAMPLE = '.. /input/siim-acr-pneumothorax-segmentation/sample_submission.csv'
items_test = Path(TEST ).ls()
ids_test = [p.stem for p in items_test]
preds_test = torch.load(os.path.join(DATA,'preds_test.pt'))
preds_test0 = preds_test.clone()
preds_test[preds_test.view(preds_test.shape[0],-1 ).float().sum(-1)< noise_th,...] = 0<save_to_csv>
train[nom_cols].nunique()
preds_t =(preds_test>best_thr ).long().numpy()
rles = []
for p in progress_bar(preds_t):
if(p.sum() > 0):
im = PIL.Image.fromarray(( p.T*255 ).astype(np.uint8)).resize(( 1024,1024))
im = np.asarray(im)
 rles.append(mask2rle(im, 1024, 1024))
else: rles.append('-1')
 sub_df = pd.DataFrame({'ImageId': ids_test, 'EncodedPixels': rles})
 sub_df.to_csv('submission.csv', index=False)
 print(len(sub_df))
sub_df.head()<categorify>
test[nom_cols].nunique()
def split_mask(mask):
threshold_obj = 30
labled,n_objs = ndimage.label(mask > best_thr)
 result = []
n_pixels = []
for i in range(n_objs):
obj =(labled == i + 1 ).astype(int)
 if(obj.sum() > threshold_obj):
result.append(obj)
 n_pixels.append(obj.sum())
 result = [x for _,x in sorted(zip(n_pixels,result),reverse=True,key=lambda x:x[0])]
return result<normalization>
train[train.nom_7=='x']
rles = []
ids_c = []
for p,idx in progress_bar(zip(preds_test,ids_test), total=len(preds_test)) :
if(p.sum() > 0):
masks = split_mask(to_np(p))
for mask in masks:
ids_c.append(idx)
 im = PIL.Image.fromarray(( mask.T*255 ).astype(np.uint8)).resize(( 1024,1024))
im = np.asarray(im)
 rles.append(mask2rle(im, 1024, 1024))
if len(masks)== 0:
rles.append('-1')
 ids_c.append(idx)
 else:
rles.append('-1')
 ids_c.append(idx)
 sub_df = pd.DataFrame({'ImageId': ids_c, 'EncodedPixels': rles})
 sub_df.to_csv('submission_split.csv', index=False)
 print(len(sub_df))
sub_df.head()<load_from_csv>
train=train[train.nom_7!='x']
df_sub_list = [pd.read_csv(f)for f in glob(os.path.join("/", "kaggle", "input", "*", "*.csv")) ]<count_unique_values>
train[train.nom_7=='x']
iid_list = df_sub_list[0]["ImageId"].unique()
print(f"{len(iid_list)} unique image IDs." )<define_variables>
labelEnc=LabelEncoder()
min_solutions = 3
assert(min_solutions >= 1 and min_solutions <= len(df_sub_list)) , \
"min_solutions has to be a number between 1 and the number of submission files"<create_dataframe>
for col in nom_cols:
train[col]=labelEnc.fit_transform(train[col])
 test[col]=labelEnc.fit_transform(test[col])
df_avg_sub = pd.DataFrame(columns=["ImageId", "EncodedPixels"])
 df_avg_sub_idx = 0
for iid in tqdm_notebook(iid_list):
avg_mask = np.zeros(( 1024,1024))
for df_sub in df_sub_list:
rles = df_sub.loc[df_sub["ImageId"]==iid, "EncodedPixels"]
for rle in rles:
if "-1" not in str(rle):
avg_mask += rle2mask(rle, 1024, 1024)/ float(len(df_sub_list))
avg_mask =(avg_mask >=(min_solutions * 255./ float(len(df_sub_list)))).astype("uint8")
 avg_rle_list = []
if avg_mask.max() > 0:
labeled_avg_mask, n_labels = skimage.measure.label(avg_mask, return_num=True)
 for label in range(1, n_labels+1):
avg_rle = mask2rle(( 255 *(labeled_avg_mask == label)).astype("uint8"), 1024, 1024)
 avg_rle_list.append(avg_rle)
 else:
avg_rle_list.append("-1")
 for avg_rle in avg_rle_list:
df_avg_sub.loc[df_avg_sub_idx] = [iid, avg_rle]
df_avg_sub_idx += 1<count_unique_values>
for col in ['bin_3', 'bin_4']:
train[col]=labelEnc.fit_transform(train[col])
 test[col]=labelEnc.fit_transform(test[col])
df_avg_sub["ImageId"].nunique()<save_to_csv>
X_temp=train.drop('target', axis=1)
 Y=train.target
df_avg_sub.to_csv("average_submission.csv", index=False )<load_from_csv>
from sklearn.mixture import GaussianMixture
train_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )<remove_duplicates>
gm = GaussianMixture(n_components=4)
 gm.fit(X_temp )
print(len(train_df))
train_df = train_df.drop_duplicates('text', keep='last')
 print(len(train_df))<count_values>
X_temp['Gaussian_Mixture']=gm.predict(X_temp )
train_df['target'].value_counts()<categorify>
test['Gaussian_Mixture']=gm.predict(test )
def preprocess_text(text):
text = re.sub(r"http\S+", "", text)
 return text<install_modules>
X_oh_temp=pd.get_dummies(X_temp.drop('id', axis=1), columns=X_temp.drop('id', axis=1 ).columns, drop_first=True, sparse=True )
!pip install -U tensorflow_text==2.3<install_modules>
X_oh=scipy.sparse.csr_matrix(X_oh_temp.values )
!pip install -q tf-models-official==2.3.0<import_modules>
test_oh_temp=pd.get_dummies(test.drop('id', axis=1), columns=test.drop('id', axis=1 ).columns, drop_first=True, sparse=True )
import tensorflow as tf
import tensorflow_hub as hub
import tensorflow_text as text
from official.nlp import optimization<split>
test_oh=scipy.sparse.csr_matrix(test_oh_temp.values )
X_train, X_valid, y_train, y_valid = train_test_split(train_df['text'].tolist() ,\
train_df['target'].tolist() ,\
test_size=0.01,\
stratify = train_df['target'].tolist() ,\
random_state=0)
<prepare_x_and_y>
X_train, X_valid, Y_train, Y_valid = train_test_split(X_oh,
Y,
test_size = 0.20,
random_state=42)
batch_size = 32
seed = 42
train_ds = tf.data.Dataset.from_tensor_slices(( X_train,y_train)).batch(batch_size)
 valid_ds = tf.data.Dataset.from_tensor_slices(( X_valid,y_valid)).batch(batch_size)
<define_variables>
lr=LogisticRegression(C=0.15, solver="lbfgs", tol=0.00005, max_iter=10000)
 lr.fit(X_train, Y_train )
bert_model_name = 'bert_en_uncased_L-12_H-768_A-12'
map_name_to_handle = {
'bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3',
'bert_en_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/3',
'bert_multi_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/3',
'small_bert/bert_en_uncased_L-2_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1',
'small_bert/bert_en_uncased_L-2_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-256_A-4/1',
'small_bert/bert_en_uncased_L-2_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-512_A-8/1',
'small_bert/bert_en_uncased_L-2_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-768_A-12/1',
'small_bert/bert_en_uncased_L-4_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-128_A-2/1',
'small_bert/bert_en_uncased_L-4_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-256_A-4/1',
'small_bert/bert_en_uncased_L-4_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1',
'small_bert/bert_en_uncased_L-4_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-768_A-12/1',
'small_bert/bert_en_uncased_L-6_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-128_A-2/1',
'small_bert/bert_en_uncased_L-6_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-256_A-4/1',
'small_bert/bert_en_uncased_L-6_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-512_A-8/1',
'small_bert/bert_en_uncased_L-6_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-6_H-768_A-12/1',
'small_bert/bert_en_uncased_L-8_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-128_A-2/1',
'small_bert/bert_en_uncased_L-8_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-256_A-4/1',
'small_bert/bert_en_uncased_L-8_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-512_A-8/1',
'small_bert/bert_en_uncased_L-8_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-8_H-768_A-12/1',
'small_bert/bert_en_uncased_L-10_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-128_A-2/1',
'small_bert/bert_en_uncased_L-10_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-256_A-4/1',
'small_bert/bert_en_uncased_L-10_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-512_A-8/1',
'small_bert/bert_en_uncased_L-10_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-10_H-768_A-12/1',
'small_bert/bert_en_uncased_L-12_H-128_A-2':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-128_A-2/1',
'small_bert/bert_en_uncased_L-12_H-256_A-4':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-256_A-4/1',
'small_bert/bert_en_uncased_L-12_H-512_A-8':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-512_A-8/1',
'small_bert/bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-12_H-768_A-12/1',
'albert_en_base':
'https://tfhub.dev/tensorflow/albert_en_base/2',
'electra_small':
'https://tfhub.dev/google/electra_small/2',
'electra_base':
'https://tfhub.dev/google/electra_base/2',
'experts_pubmed':
'https://tfhub.dev/google/experts/bert/pubmed/2',
'experts_wiki_books':
'https://tfhub.dev/google/experts/bert/wiki_books/2',
'talking-heads_base':
'https://tfhub.dev/tensorflow/talkheads_ggelu_bert_en_base/1',
}
map_model_to_preprocess = {
'bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'bert_en_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_cased_preprocess/2',
'small_bert/bert_en_uncased_L-2_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-2_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-2_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-2_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-4_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-4_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-4_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-4_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-6_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-6_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-6_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-6_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-8_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-8_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-8_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-8_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-10_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-10_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-10_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-10_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-12_H-128_A-2':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-12_H-256_A-4':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-12_H-512_A-8':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'small_bert/bert_en_uncased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'bert_multi_cased_L-12_H-768_A-12':
'https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/2',
'albert_en_base':
'https://tfhub.dev/tensorflow/albert_en_preprocess/2',
'electra_small':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'electra_base':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'experts_pubmed':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'experts_wiki_books':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
'talking-heads_base':
'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/2',
}
tfhub_handle_encoder = map_name_to_handle[bert_model_name]
tfhub_handle_preprocess = map_model_to_preprocess[bert_model_name]
print(f'BERT model selected : {tfhub_handle_encoder}')
 print(f'Preprocess model auto-selected: {tfhub_handle_preprocess}' )<categorify>
y_pred_lr=lr.predict_proba(X_valid )
def build_classifier_model() :
text_input = tf.keras.layers.Input(shape=() , dtype=tf.string, name='text')
 preprocessing_layer = hub.KerasLayer(tfhub_handle_preprocess, name='preprocessing')
 encoder_inputs = preprocessing_layer(text_input)
 encoder = hub.KerasLayer(tfhub_handle_encoder, trainable=True, name='BERT_encoder')
 outputs = encoder(encoder_inputs)
 net = outputs['pooled_output']
net = tf.keras.layers.Dropout(0.1 )(net)
 net = tf.keras.layers.Dense(8,activation ="relu",name ="hidden" )(net)
 net = tf.keras.layers.Dense(1, activation= "sigmoid" , name='classifier' )(net)
 return tf.keras.Model(text_input, net)
<choose_model_class>
roc_auc_score(Y_valid.values, y_pred_lr[:,1] )
loss = tf.keras.losses.BinaryCrossentropy(from_logits=True)
 metrics = tf.metrics.BinaryAccuracy()<choose_model_class>
lr.fit(X_oh, Y )
epochs = 30
steps_per_epoch = tf.data.experimental.cardinality(train_ds ).numpy()
num_train_steps = steps_per_epoch * epochs
num_warmup_steps = int(0.1*num_train_steps)
 init_lr = 2e-5
optimizer = optimization.create_optimizer(init_lr=init_lr,
num_train_steps=num_train_steps,
num_warmup_steps=num_warmup_steps,
optimizer_type='adamw')
 classifier_model.compile(optimizer=optimizer,
loss=loss,
metrics=metrics )<train_model>
y_pred=lr.predict_proba(X_oh)
 roc_auc_score(Y.values, y_pred[:,1] )
print(f'Training model with {tfhub_handle_encoder}')
 history = classifier_model.fit(x=train_ds, validation_data=valid_ds, epochs=epochs)
<predict_on_test>
test_pred_lr=lr.predict_proba(test_oh )
probs = classifier_model.predict(test_df["text"])
 threshold = 0.4
preds = np.where(probs[:,] > threshold, 1, 0 )<load_from_csv>
output_dict = {'id': test.id,
'target': test_pred_lr[:,1]}
output = pd.DataFrame(output_dict, columns = ['id', 'target'])
 output.head(10 )
<prepare_output><EOS> 
output.to_csv('submission_One_Hot.csv', index=False )