Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<define_variables>
!pip install transformers==3.5.1
!pip install pyspellchecker
all_results = []
all_start_end = []
base_model_type = "roberta-base"
CURR_PATH = ".. /input/"
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=os.path.join(CURR_PATH, "transformers-vocab/{}-vocab.json".format(base_model_type)) ,
merges_file=os.path.join(CURR_PATH, "transformers-vocab/{}-merges.txt".format(base_model_type)) ,
lowercase=True,
add_prefix_space=True 
)
 for i in range(len(start_logits_word_level)) :
start_idx_token, end_idx_token = get_token_level_idx(roberta_base_start_logits_token_level[i], \
roberta_base_end_logits_token_level[i], \
start_logits_word_level[i], \
end_logits_word_level[i], \
roberta_base_word_level_bbx[i] 
)
 all_start_end.append(( start_idx_token, end_idx_token))
_, final_text = calculate_jaccard_score(
 original_tweet=tweets[i],
selected_text=tweets[i],
idx_start=start_idx_token,
idx_end=end_idx_token,
model_type=base_model_type,
tweet_offsets=roberta_base_token_level_offsets[i],
tokenizer=tokenizer,
sentiment=sentiments[i] 
)
 all_results.append(final_text )<load_from_csv>
import pandas as pd
import torchtext
from transformers import BertTokenizer, BertForMaskedLM, BertConfig
import transformers
import torch
from torch.utils.data import Dataset, DataLoader
from torch import optim
from torch import cuda
from sklearn.model_selection import train_test_split
import re
import string
submission = pd.read_csv(os.path.join("/kaggle/input/tweet-sentiment-extraction", "sample_submission.csv"))
test = pd.read_csv(os.path.join("/kaggle/input/tweet-sentiment-extraction", "test.csv"))<feature_engineering>
train_val_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )
for i in range(len(submission)) :
if test['sentiment'][i] == 'neutral' or len(test['text'][i].split())< 4:
submission.loc[i, 'selected_text'] = test['text'][i]
else:
submission.loc[i, 'selected_text'] = all_results[i]<feature_engineering>
train_val_df = train_val_df.loc[:,["text","target"]]
test_df = test_df.loc[:,["text"]]
test_df["target"] = [0]*len(test_df["text"] )
submission["text"] = test["text"]<categorify>
print(train_val_df)
 print(test_df.head())
 original_df = train_val_df.copy()
def reverse_preprocessing(text):
text = text.replace(".... ", ".... ")
 text = text.replace("...", "...")
 text = text.replace(".. ", ".. ")
 text = text.replace("! ! ! !", "!!!!")
 text = text.replace("! ! !", "!!!")
 text = text.replace("! !", "!!")
 text = text.replace("? ? ? ?", "????")
 text = text.replace("? ? ?", "???")
 text = text.replace("? ?", "??")
 return text
def find_text_idx(text, selected_text):
text_len = len(text)
 for start_idx in range(text_len):
if text[start_idx] == selected_text[0]:
for end_idx in range(start_idx+1, text_len+1):
contained_text = "".join(text[start_idx: end_idx].split())
 if contained_text == "".join(selected_text.split()):
return start_idx, end_idx
return None, None
def calculate_spaces(text, selected_text):
selected_text = " ".join(selected_text.split())
 start_idx, end_idx = find_text_idx(text, selected_text)
 if start_idx is None:
start_idx = 0
print("----------------- error no start idx find ------------------")
 print("text:", text, "prediction:", selected_text)
 print("----------------- error no start idx find ------------------")
 if end_idx is None:
end_idx = len(text)
 print("----------------- error no end idx find ------------------")
 print("text:", text, "prediction:", selected_text)
 print("----------------- error no end idx find ------------------")
 x = text[:start_idx]
try:
if x[-1] == " ":
x = x[:-1]
except:
pass
l1 = len(x)
 l2 = len(" ".join(x.split()))
return l1 - l2, start_idx, end_idx
def pp_v2(text, predicted):
text = str(text ).lower()
predicted = predicted.lower()
predicted = predicted.strip()
if len(predicted)== 0:
return predicted
predicted = reverse_preprocessing(str(predicted))
spaces, index_start, index_end = calculate_spaces(text, predicted)
 if spaces == 1:
if len(text[max(0, index_start-1): index_end+1])<= 0 or text[max(0, index_start-1): index_end+1][-1] != ".":
return text[max(0, index_start - 1): index_end]
else:
return text[max(0, index_start-1): index_end+1]
elif spaces == 2:
return text[max(0, index_start-2): index_end]
elif spaces == 3:
return text[max(0, index_start-3): index_end-1]
elif spaces == 4:
return text[max(0, index_start-4): index_end-2]
else:
return predicted<compute_test_metric>
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"couldnt" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"doesnt" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"havent" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"shouldnt" : "should not",
"that's" : "that is",
"thats" : "that is",
"there's" : "there is",
"theres" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"theyre": "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not"}
def replace_typical_misspell(text):
text = text.lower()
mispellings_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
def replace(match):
return mispell_dict[match.group(0)]
return mispellings_re.sub(replace, text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x : replace_typical_misspell(x))
test_df['text'] = test_df['text'].apply(lambda x : replace_typical_misspell(x))
def remove_space(string):
string = BeautifulSoup(string ).text.strip().lower()
string = re.sub(r'(( http)\S+)', 'http', string)
 string = re.sub(r'@\w*', '', string)
 string = re.sub(r'
string = re.sub(r'\s+', ' ', string)
 return string
train_val_df['text'] = train_val_df['text'].apply(lambda x : remove_space(x))
test_df['text'] = test_df['text'].apply(lambda x : remove_space(x))
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 train_val_df['text'] = train_val_df['text'].apply(lambda x: remove_emoji(x))
test_df['text'] = test_df['text'].apply(lambda x: remove_emoji(x))
submission["new_selected"] = submission.apply(lambda x: pp_v2(x.text, x.selected_text), axis=1 )<feature_engineering>
test_df.to_csv("test.tsv", sep='\t', index=False, header=None)
 print(test_df.shape)
 train_val_df.to_csv("train_eval.tsv", sep='\t', index=False, header=None)
 print(train_val_df.shape )
submission.selected_text = submission["new_selected"]<save_to_csv>
max_length = 50
def tokenizer_50(input_text):

 return tokenizer.encode(input_text, max_length=50, return_tensors='pt')[0]
TEXT = torchtext.data.Field(sequential=True, tokenize=tokenizer_50, use_vocab=False, lower=False,
include_lengths=True, batch_first=True, fix_length=max_length, pad_token=0)
 LABEL = torchtext.data.Field(sequential=False, use_vocab=False )
submission[["textID","selected_text"]].to_csv("submission.csv", index=False )<set_options>
tokenizer = BertTokenizer.from_pretrained('bert-base-cased' )
numpy.set_printoptions(threshold=sys.maxsize)
<load_from_csv>
dataset_train_eval, dataset_test = torchtext.data.TabularDataset.splits(path='.', train='./train_eval.tsv', test='./test.tsv', format='tsv', fields=[('Text', TEXT),('Label', LABEL)] )
train= pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 test= pd.read_csv('.. /input/Kannada-MNIST/test.csv', index_col='id')
 print('Train shape: ', train.shape)
 print('Train missing vals: ', train.isna().sum().sum())
 print('Test shape: ', test.shape)
 print('Test missing vals: ', test.isna().sum().sum() )<prepare_x_and_y>
dataset_train, dataset_eval = dataset_train_eval.split(
 split_ratio=1.0 - 1800/7613, random_state=random.seed(1234))
print(dataset_train.__len__())
 print(dataset_eval.__len__())
 print(dataset_test.__len__() )
x_train= train.iloc[:, 1:].values
y_train= train.iloc[:, 0].values
x_test= test.iloc[:, :].values<split>
print(tokenizer.convert_ids_to_tokens(item.Text.tolist()))
print(int(item.Label))
x_train= x_train.reshape(train.shape[0], 28, 28, 1)
 x_test= x_test.reshape(test.shape[0], 28, 28, 1)
 y_train= to_categorical(y_train, 10)
 x_train, x_valid, y_train, y_valid= train_test_split(x_train, y_train, test_size= 0.2, random_state= 42 )<choose_model_class>
batch_size = 32
dl_train = torchtext.data.Iterator(
 dataset_train, batch_size=batch_size, train=True)
 dl_eval = torchtext.data.Iterator(
 dataset_eval, batch_size=batch_size, train=False, sort=False)
 dl_test = torchtext.data.Iterator(
 dataset_test, batch_size=batch_size, train=False, sort=False)
 dataloaders_dict = {"train": dl_train, "val": dl_eval}
<choose_model_class>
model = BertModel.from_pretrained('bert-base-cased' )
num_classes= 10
lr= 0.001
batch_size= 1024
model= Sequential()
model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu', input_shape=(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size=5, padding='same', strides= 2, activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.4))
model.add(Conv2D(64, kernel_size= 3, padding='same', activation= 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size= 3, padding='same', activation= 'relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size= 5, padding= 'same', strides= 2, activation= 'relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(128))
model.add(BatchNormalization())
 model.add(Dense(10, activation='softmax'))
model.summary()
optimizer = RMSprop(lr=lr)
 model.compile(loss='categorical_crossentropy', optimizer= optimizer, metrics= ['accuracy'] )<train_model>
class BertForTwitter(nn.Module):
def __init__(self):
super(BertForTwitter, self ).__init__()
self.bert = model
self.cls = nn.Linear(in_features=768, out_features=2)
 nn.init.normal_(self.cls.weight, std=0.02)
 nn.init.normal_(self.cls.bias, 0)
 def forward(self, input_ids):

 result = self.bert(input_ids)
 vec_0 = result[0]
vec_0 = vec_0[:, 0, :]
vec_0 = vec_0.view(-1, 768)
 output = self.cls(vec_0)
 return output
img_gen= ImageDataGenerator(rotation_range= 5, zoom_range= 0.5, horizontal_flip= 0, vertical_flip= 0,
width_shift_range= 2, height_shift_range= 2, rescale= 1/255)
 valid_img_gen = ImageDataGenerator(rescale=1./255.)
learning_rate_reduction = ReduceLROnPlateau(monitor='val_loss', patience=200, verbose=1, factor=0.2)
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=200)
 model.fit_generator(img_gen.flow(x_train, y_train, batch_size= batch_size), steps_per_epoch= 100, epochs= 100,
validation_data= valid_img_gen.flow(x_valid, y_valid, batch_size= batch_size), validation_steps= 50,
callbacks=[learning_rate_reduction, es], verbose= 1 )<save_to_csv>
net = BertForTwitter()
net.train()
print('ネットワーク設定完了' )
pred= model.predict_classes(x_test/255)
 sub= pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 sub['label']= pred
sub.to_csv('submission.csv', index= False )<choose_model_class>
for param in net.parameters() :
param.requires_grad = False
for param in net.bert.encoder.layer[-1].parameters() :
param.requires_grad = True
for param in net.cls.parameters() :
param.requires_grad = True
def cnn_model() :
inp = tf.keras.Input(shape=(28,28,1))
x1 = tf.keras.layers.Conv2D(128,(1,1), strides=(1,1), activation='relu' )(inp)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x3 = tf.keras.layers.Conv2D(128,(3,3), padding='same', strides=(1,1), activation='relu' )(inp)
 x3 = tf.keras.layers.BatchNormalization()(x3)
 x5 = tf.keras.layers.Conv2D(128,(5,5), padding='same', strides=(1,1), activation='relu' )(inp)
 x5 = tf.keras.layers.BatchNormalization()(x5)
 averaged = tf.keras.layers.Average()([x1,x3,x5])
 averaged = tf.keras.layers.Activation('relu' )(averaged)
 averaged = tf.keras.layers.BatchNormalization()(averaged)
 x = tf.keras.layers.Conv2D(128,(5,5), strides=(1,1), activation='relu' )(averaged)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.MaxPooling2D()(x)
 x = tf.keras.layers.Dropout(rate=0.25 )(x)
 first_block_out = tf.keras.layers.Flatten()(x)
 x1 = tf.keras.layers.Conv2D(128,(1,1), strides=(1,1), activation='relu' )(x)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x3 = tf.keras.layers.Conv2D(128,(3,3), padding='same', strides=(1,1), activation='relu' )(x)
 x3 = tf.keras.layers.BatchNormalization()(x3)
 x5 = tf.keras.layers.Conv2D(128,(5,5), padding='same', strides=(1,1), activation='relu' )(x)
 x5 = tf.keras.layers.BatchNormalization()(x5)
 averaged = tf.keras.layers.Average()([x1,x3,x5])
 averaged = tf.keras.layers.Activation('relu' )(averaged)
 averaged = tf.keras.layers.BatchNormalization()(averaged)
 x = tf.keras.layers.Conv2D(128,(3,3), strides=(1,1), activation='relu' )(averaged)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.MaxPooling2D()(x)
 x = tf.keras.layers.Dropout(rate=0.25 )(x)
 second_block_out = tf.keras.layers.Flatten()(x)
 x1 = tf.keras.layers.Conv2D(256,(1,1), strides=(1,1), activation='relu' )(x)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x3 = tf.keras.layers.Conv2D(256,(3,3), padding='same', strides=(1,1), activation='relu' )(x)
 x3 = tf.keras.layers.BatchNormalization()(x3)
 x5 = tf.keras.layers.Conv2D(256,(5,5), padding='same', strides=(1,1), activation='relu' )(x)
 x5 = tf.keras.layers.BatchNormalization()(x5)
 averaged = tf.keras.layers.Average()([x1,x3,x5])
 averaged = tf.keras.layers.Activation('relu' )(averaged)
 averaged = tf.keras.layers.BatchNormalization()(averaged)
 x = tf.keras.layers.Conv2D(256,(1,1), strides=(1,1), activation='relu' )(averaged)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.MaxPooling2D()(x)
 x = tf.keras.layers.Dropout(rate=0.25 )(x)
 third_block_out = tf.keras.layers.Flatten()(x)
 x1 = tf.keras.layers.Conv2D(256,(1,1), strides=(1,1), activation='relu' )(x)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x3 = tf.keras.layers.Conv2D(256,(3,3), padding='same', strides=(1,1), activation='relu' )(x)
 x3 = tf.keras.layers.BatchNormalization()(x3)
 x5 = tf.keras.layers.Conv2D(256,(5,5), padding='same', strides=(1,1), activation='relu' )(x)
 x5 = tf.keras.layers.BatchNormalization()(x5)
 averaged = tf.keras.layers.Average()([x1,x3,x5])
 averaged = tf.keras.layers.Activation('relu' )(averaged)
 averaged = tf.keras.layers.BatchNormalization()(averaged)
 x = tf.keras.layers.Conv2D(256,(1,1), strides=(1,1), activation='relu' )(averaged)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.MaxPooling2D()(x)
 x = tf.keras.layers.Dropout(rate=0.25 )(x)
 fourth_block_out = tf.keras.layers.Flatten()(x)
 print(first_block_out.shape, second_block_out.shape, third_block_out.shape, fourth_block_out.shape)
 x = tf.keras.layers.Concatenate()([second_block_out, third_block_out, fourth_block_out])
 x = tf.keras.layers.Dense(512, activation=tf.nn.relu )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dense(256, activation=tf.nn.relu )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dense(64, activation=tf.nn.relu )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 output = tf.keras.layers.Dense(10, activation=tf.nn.softmax )(x)
 model = tf.keras.Model(inputs=inp, outputs=output)
 return model
train_file = "/kaggle/input/Kannada-MNIST/train.csv"
validate_file = "/kaggle/input/Kannada-MNIST/Dig-MNIST.csv"
test_file = "/kaggle/input/Kannada-MNIST/test.csv"
data = pd.read_csv(train_file)
 data_numpy = np.array(data)
 print(data_numpy.shape)
 data_numpy_label = data_numpy[:,0]
data_numpy_image = data_numpy[:,1:]
data_numpy_label = data_numpy_label.reshape(60000, 1)
 print(data_numpy_label.shape)
 print(data_numpy_image.shape)
 data_numpy_image_28x28 = data_numpy_image.reshape(60000, 28, 28)
 print(data_numpy_image_28x28.shape)
 print(type(data_numpy_image_28x28[0][0][0]))
data_numpy_image_28x28 = data_numpy_image_28x28.astype(np.uint8)
 print(type(data_numpy_image_28x28[0][0][0]))
validate_data = pd.read_csv(validate_file)
 validate_data_numpy = np.array(validate_data)
 print(validate_data_numpy.shape)
 validate_data_numpy_label = validate_data_numpy[:,0]
validate_data_numpy_image = validate_data_numpy[:,1:]
validate_data_numpy_label = validate_data_numpy_label.reshape(10240, 1)
 print(validate_data_numpy_label.shape)
 print(validate_data_numpy_image.shape)
 validate_data_numpy_image_28x28 = validate_data_numpy_image.reshape(10240, 28, 28)
 print(validate_data_numpy_image_28x28.shape)
 validate_data_numpy_image = validate_data_numpy_image.astype(np.float32)
 validate_data_numpy_label = validate_data_numpy_label.astype(np.float32)
 test_data = pd.read_csv(test_file)
 test_data_numpy = np.array(test_data)
 print(test_data_numpy.shape)
 test_data_numpy_label = test_data_numpy[:,0]
test_data_numpy_image = test_data_numpy[:,1:]
test_data_numpy_label = test_data_numpy_label.reshape(-1, 1)
 print(test_data_numpy_label.shape)
 print(test_data_numpy_image.shape)
 test_data_numpy_image_28x28 = test_data_numpy_image.reshape(-1, 28, 28)
 print(test_data_numpy_image_28x28.shape)
 test_data_numpy_image = test_data_numpy_image.astype(np.float32)
 test_data_numpy_label = test_data_numpy_label.astype(np.float32)
 data_numpy_image_28x28 = data_numpy_image_28x28.astype(np.float32)
 data_numpy_image = data_numpy_image.astype(np.float32)
 reg = LinearSVC(random_state=0, tol=1e-5 ).fit(data_numpy_image, data_numpy_label)
 score = reg.score(data_numpy_image, data_numpy_label)
 print(score)
 print(reg.get_params())
 predict_label = reg.predict(validate_data_numpy_image)
 print(predict_label.shape)
 correct = 0
total = 0
for i in range(len(predict_label)) :
pred_label = predict_label[i]
ground_truth_label = validate_data_numpy_label[i][0]
if int(round(pred_label)) == int(ground_truth_label):
correct += 1
total += 1
print("correct / total, %", correct, total, correct*100./total, "%")
 test_label = reg.predict(test_data_numpy_image)
 print(test_label.shape)
 x_inp = Input(shape=(28,28,1))
x = Conv2D(filters=64, kernel_size=(( 3,3)) , strides=(1, 1), padding='valid', activation='tanh' )(x_inp)
 x = Conv2D(filters=128, kernel_size=(3,3), strides=(1, 1), padding='valid', activation='tanh' )(x)
 x = Conv2D(filters=256, kernel_size=(3,3), strides=(1, 1), padding='valid', activation='tanh' )(x)
 x = Conv2D(filters=256, kernel_size=(3,3), strides=(1, 1), padding='valid', activation='tanh' )(x)
 x_flat = Flatten()(x)
 x_flat = Dense(256, activation='tanh' )(x_flat)
 x_flat = Dropout(rate=0.3 )(x_flat)
 x_flat = Dense(256, activation='tanh' )(x_flat)
 x_flat = Dropout(rate=0.3 )(x_flat)
 x4 = Dense(10, activation='softmax' )(x_flat)
 model = cnn_model()
data_numpy_label_cate = keras.utils.to_categorical(data_numpy_label, num_classes=10, dtype='float32')
 data_numpy_image_28x28 = np.expand_dims(data_numpy_image_28x28, axis=-1)
 print(data_numpy_image_28x28.shape, data_numpy_image_28x28.dtype)
 print(data_numpy_label_cate.shape, data_numpy_label_cate.dtype)
 print(model.input_shape, model.output_shape)
 model.compile(optimizer="Adam", loss="categorical_crossentropy", metrics=['accuracy', 'mse'])
 model.fit(x=data_numpy_image_28x28, y=data_numpy_label_cate, batch_size=64, epochs=15, verbose=1, validation_split=0.1)
 validate_data_numpy_image_28x28 = validate_data_numpy_image_28x28.astype(np.float32)
 validate_data_numpy_image_28x28 = np.expand_dims(validate_data_numpy_image_28x28, axis=-1)
 validate_data_numpy_label_cate = keras.utils.to_categorical(validate_data_numpy_label, num_classes=10, dtype='float32')
 print(validate_data_numpy_image_28x28.shape, validate_data_numpy_image_28x28.dtype)
 print(validate_data_numpy_label_cate.shape, validate_data_numpy_label_cate.dtype)
 resultss_validation=model.evaluate(validate_data_numpy_image_28x28, validate_data_numpy_label_cate, verbose=1)
 print(model.metrics_names)
 print(resultss_validation)
 test_data_numpy_image_28x28 = test_data_numpy_image_28x28.astype(np.float32)
 test_data_numpy_image_28x28_expnd = np.expand_dims(test_data_numpy_image_28x28, axis=-1)
 test_label = model.predict(test_data_numpy_image_28x28_expnd)
 print(test_label.shape)
 result = []
for i in range(len(test_label)) :
pred_label = np.argmax(test_label[i])
 output_got = int(round(pred_label))
result.append([i, output_got])
 result = np.append(np.array([['id', 'label']]), result, axis=0)
 print(result.shape)
 with open("submission.csv", 'w', newline='')as csvfile:
spamwriter = csv.writer(csvfile)
 for r in result:
spamwriter.writerow(r)
<set_options>
optimizer = optim.Adam([
{'params': net.bert.encoder.layer[-1].parameters() , 'lr': 5e-5},
{'params': net.cls.parameters() , 'lr': 1e-4}
])
 criterion = nn.CrossEntropyLoss()
%matplotlib inline 
<choose_model_class>
def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):
max_acc = 0
Stop_flag = False
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print("使用デバイス：", device)
 print('-----start-------')
 net.to(device)
 torch.backends.cudnn.benchmark = True
batch_size = dataloaders_dict["train"].batch_size
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
net.train()
else:
net.eval()
epoch_loss = 0.0
epoch_corrects = 0
iteration = 1
for batch in(dataloaders_dict[phase]):
inputs = batch.Text[0].to(device)
 labels = batch.Label.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = net(inputs)
 loss = criterion(outputs, labels)
 _, preds = torch.max(outputs, 1)
 if phase == 'train':
loss.backward()
optimizer.step()
if(iteration % 50 == 0):
acc =(torch.sum(preds == labels.data)
 ).double() /batch_size
print('イテレーション {} || Loss: {:.4f} || 10iter.|| 本イテレーションの正解率：{}'.format(
 iteration, loss.item() , acc))
iteration += 1
epoch_loss += loss.item() * batch_size
epoch_corrects += torch.sum(preds == labels.data)
 epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_acc = epoch_corrects.double(
)/ len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} Acc: {:.4f}'.format(epoch+1, num_epochs,
phase, epoch_loss, epoch_acc))
if phase == "val":
if epoch_acc < max_acc:
count += 1
if count >= 3:
Stop_flag = True
else:
count = 0
max_acc = epoch_acc
print(count)
 if Stop_flag:
break
return net
def build_model(optimizer=Adam()):
model = Sequential()
model.add(Conv2D(64, kernel_size=3, padding='same', activation='relu', input_shape=(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=3, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=5, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(128, kernel_size=3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(128, kernel_size=3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(128, kernel_size=5, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.2))
model.add(Conv2D(256, kernel_size=3, activation='relu'))
model.add(BatchNormalization())
 model.add(GlobalMaxPooling2D())
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(1024))
model.add(BatchNormalization())
 model.add(Dense(1024))
model.add(BatchNormalization())
 model.add(Dense(512))
model.add(BatchNormalization())
 model.add(Dense(256))
model.add(BatchNormalization())
 model.add(Dense(128))
model.add(BatchNormalization())
 model.add(Dense(10, activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
 return model<choose_model_class>
num_epochs = 50
net_trained = train_model(net, dataloaders_dict,
criterion, optimizer, num_epochs=num_epochs )
model = build_model(Adam(learning_rate=1e-3))<load_pretrained>
sample_submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = ans_list
sample_submission
<load_from_csv><EOS> 
sample_submission.to_csv("submission_plus.csv", index=False )