Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<load_pretrained>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
class TweetModel(transformers.BertPreTrainedModel):

 def __init__(self, conf):
super(TweetModel, self ).__init__(conf)
 self.bert = transformers.BertModel.from_pretrained(config.BERT_PATH, config=conf)
 self.drop_out = nn.Dropout(0.1)
 self.l0 = nn.Linear(768 * 2, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self, ids, mask, token_type_ids):
outputs = self.bert(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 out = torch.cat(( outputs.hidden_states[-1], outputs.hidden_states[-2]), dim=-1)
 out = self.drop_out(out)
 logits = self.l0(out)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<compute_test_metric>
mnist_train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 mnist_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )
def loss_fn(start_logits, end_logits, start_positions, end_positions):

 loss_fct = nn.CrossEntropyLoss()
start_loss = loss_fct(start_logits, start_positions)
 end_loss = loss_fct(end_logits, end_positions)
 total_loss =(start_loss + end_loss)
 return total_loss<choose_model_class>
img_rows, img_cols = 28, 28
num_classes = 10
def data_prep(raw):
out_y = keras.utils.np_utils.to_categorical(raw.label, num_classes)
 num_images = raw.shape[0]
x_as_array = raw.values[:,1:]
x_shaped_array = x_as_array.reshape(num_images, img_rows, img_cols, 1)
 out_x = x_shaped_array / 255
return out_x, out_y
x, y = data_prep(mnist_train)
def train_fn(data_loader, model, optimizer, device, scheduler=None):

 model.train()
losses = utils.AverageMeter()
jaccards = utils.AverageMeter()
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
offsets = d["offsets"]
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 model.zero_grad()
outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 loss.backward()
optimizer.step()
scheduler.step()
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg )<compute_test_metric>
X_train, X_val, y_train, y_val = train_test_split(x, y, test_size=0.10, random_state=42 )
def calculate_jaccard_score(
 original_tweet,
target_string,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):

 if idx_end < idx_start:
idx_end = idx_start
filtered_output = ""
for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if sentiment_val == "neutral" or len(original_tweet.split())< 2:
filtered_output = original_tweet
jac = utils.jaccard(target_string.strip() , filtered_output.strip())
 return jac, filtered_output
def eval_fn(data_loader, model, device):

 model.eval()
losses = utils.AverageMeter()
jaccards = utils.AverageMeter()
with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg)
 print(f"Jaccard = {jaccards.avg}")
 return jaccards.avg<load_from_csv>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
def run(fold):

 dfx = pd.read_csv(config.TRAINING_FILE)
 df_train = dfx[dfx.kfold != fold].reset_index(drop=True)
 df_valid = dfx[dfx.kfold == fold].reset_index(drop=True)
 train_dataset = TweetDataset(
 tweet=df_train.text.values,
sentiment=df_train.sentiment.values,
selected_text=df_train.selected_text.values 
)
 train_data_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=config.TRAIN_BATCH_SIZE,
num_workers=4 
)
 valid_dataset = TweetDataset(
 tweet=df_valid.text.values,
sentiment=df_valid.sentiment.values,
selected_text=df_valid.selected_text.values 
)
 valid_data_loader = torch.utils.data.DataLoader(
 valid_dataset,
batch_size=config.VALID_BATCH_SIZE,
num_workers=2 
)
 device = torch.device("cuda")
 model_config = transformers.BertConfig.from_pretrained(config.BERT_PATH)
 model_config.output_hidden_states = True
model = TweetModel(conf=model_config)
 model.to(device)
 num_train_steps = int(len(df_train)/ config.TRAIN_BATCH_SIZE * config.EPOCHS)
 param_optimizer = list(model.named_parameters())
 no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
optimizer_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0},
]
optimizer = AdamW(optimizer_parameters, lr=3e-5)
 scheduler = get_linear_schedule_with_warmup(
 optimizer,
num_warmup_steps=0,
num_training_steps=num_train_steps 
)
 es = utils.EarlyStopping(patience=2, mode="max")
 print(f"Training is Starting for fold={fold}")
 for epoch in range(3):
train_fn(train_data_loader, model, optimizer, device, scheduler=scheduler)
 jaccard = eval_fn(valid_data_loader, model, device)
 print(f"Jaccard Score = {jaccard}")
 es(jaccard, model, model_path=f"model_{fold}.bin")
 if es.early_stop:
print("Early stopping")
 break<load_from_csv>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size=(3, 3),padding = 'Same',
activation='relu',
input_shape=(img_rows, img_cols, 1)))
 model.add(BatchNormalization(axis=1))
model.add(Conv2D(filters = 32, kernel_size=(3, 3),padding = 'Same', activation='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.4))
model.add(BatchNormalization(axis=1))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(BatchNormalization(axis=1))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(BatchNormalization(axis=1))
model.add(Dense(128, activation='relu'))
model.add(BatchNormalization(axis=1))
model.add(Dropout(0.4))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer='adam',
metrics=['accuracy'])
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1)
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', patience=3, verbose=1, factor=0.5, min_lr=0.00001)
 history = model.fit_generator(datagen.flow(X_train,y_train, batch_size=128),
epochs=22,callbacks=[learning_rate_reduction],
validation_data =(X_val,y_val))
df_test = pd.read_csv(".. /input/tweet-sentiment-extraction/test.csv")
 df_test.loc[:, "selected_text"] = df_test.text.values<load_pretrained>
x_test_shaped_array = mnist_test.values.reshape(28000, img_rows, img_cols, 1)
 x_test = x_test_shaped_array / 255
<load_pretrained><EOS> 
results = model.predict(x_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist.csv",index=False )