Kaggle competition: Tweet Sentiment Extraction
<SOS> metric: Jaccard Kaggle data source: tweet-sentiment-extraction<train_model>
print('TF version',tf.__version__ )
model_histories = []
i = 1
for model in models_ensemble:
xtrain, xtest, ytrain, ytest = train_test_split(X_train_cnn, labels, test_size = 0.07)
 print("Model " +str(i)+ " : ",end="")
 model_history = model.fit_generator(data_aug.flow(xtrain, ytrain, batch_size = 64), epochs = 1, verbose = 1, validation_data =(xtest, ytest), steps_per_epoch = xtrain.shape[0])
 model_histories.append(model_history)
 i += 1<predict_on_test>
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 EPOCHS = 1
BATCH_SIZE = 32
PAD_ID = 1
SEED = 88888
LABEL_SMOOTHING = 0.1
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 sentiment_id = {'positive': 1313, 'neutral': 7974, 'negative': 2430}
testdata = pd.read_csv(test_dir)
 testdata = testdata.values.tolist()
testdata = np.array(testdata)
 testdata_reshaped = testdata.reshape(testdata.shape[0], 28, 28, 1)
 testdata_reshaped = testdata_reshaped.astype('float')/255.0
def make_predictions_final_model(curr_model):
prediction_array = curr_model.predict_on_batch(testdata_reshaped)
 predictions = [np.argmax(i)for i in prediction_array]
return predictions<categorify>
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()
predictions_ensemble = []
for model in models_ensemble:
curr_predictions = make_predictions_final_model(model)
 predictions_ensemble.append(curr_predictions)
 prediction_per_image = []
for i in range(len(predictions_ensemble[0])) :
temppred = [predictions_ensemble[0][i], predictions_ensemble[1][i], predictions_ensemble[2][i], predictions_ensemble[3][i], predictions_ensemble[4][i], predictions_ensemble[5][i], predictions_ensemble[6][i]]
prediction_per_image.append(temppred)
 prediction_per_image = np.array(prediction_per_image)
 modes = stats.mode(prediction_per_image, axis = 1)
 final_predictions = []
for i in modes[0]:
final_predictions.append(i[0] )<save_to_csv>
stop = stopwords.words('english')
 stop += ["_________________________________", "u"]
punct = list(string.punctuation)
 punct.remove("-")
 punct.append(" " )
final_csv = []
csv_title = ['ImageId', 'Label']
final_csv.append(csv_title)
 for i in range(len(final_predictions)) :
image_id = i + 1
label = final_predictions[i]
temp = [image_id, label]
final_csv.append(temp)
 print(len(final_csv))
with open('submission_csv_aug.csv', 'w')as file:
writer = csv.writer(file)
 writer.writerows(final_csv)
 file.close()<load_from_csv>
def get_synonyms(word):

 if word.lower() in stop:
return [word], [1]
synonyms = set()
for syn in wordnet.synsets(word):
for l in syn.lemmas() :
synonym = l.name().replace("_", " " ).replace("-", " " ).lower()
synonym = "".join([char for char in synonym if char in ' qwertyuiopasdfghjklzxcvbnm'])
 synonyms.add(synonym)
 if word not in synonyms:
synonyms.add(word)
 n = len(synonyms)
 if n == 1:
word_ = "".join(list(filter(lambda x: x not in punct, word)))
 if word_.lower() in stop:
return [word, word_], [0.5, 0.5]
for syn in wordnet.synsets(word_):
for l in syn.lemmas() :
synonym = l.name().replace("_", " " ).replace("-", " " ).lower()
synonym = "".join([char for char in synonym if char in ' qwertyuiopasdfghjklzxcvbnm'])
 synonyms.add(synonym)
 if word_ not in synonyms:
synonyms.add(word_)
 n = len(synonyms)
 if n == 1:
probabilities = [1]
else:
probabilities = [0.5 if w==word else 0.5/(n-1)for w in synonyms]
return list(synonyms), probabilities
data_train = pd.read_csv(".. /input/train.csv")
 data_train.head() 
<prepare_x_and_y>
def swap_words(words):
words = words.split()
if len(words)< 2:
return " ".join(words), False
random_idx = np.random.randint(0, len(words)-1)
 words[random_idx], words[random_idx+1] = words[random_idx+1], words[random_idx]
return " ".join(words), True
y_train = data_train['label']
x_train = data_train.drop(labels = ["label"], axis = 1)
 print('Shape of whole data for training', data_train.shape)
 print('x_train:', x_train.shape)
 print('y_train:', y_train.shape)
 %matplotlib inline
def convert_to_grid(x_input):
N, H, W = x_input.shape
grid_size = int(ceil(sqrt(N)))
 grid_height = H * grid_size + 1 *(grid_size - 1)
 grid_width = W * grid_size + 1 *(grid_size - 1)
 grid = np.zeros(( grid_height, grid_width)) + 255
next_idx = 0
y0, y1 = 0, H
for y in range(grid_size):
x0, x1 = 0, W
for x in range(grid_size):
if next_idx < N:
img = x_input[next_idx]
low, high = np.min(img), np.max(img)
 grid[y0:y1, x0:x1] = 255.0 *(img - low)/(high - low)
 next_idx += 1
x0 += W + 1
x1 += W + 1
y0 += H + 1
y1 += H + 1
return grid
examples = np.array(x_train.iloc[:81] ).reshape(81, 28, 28)
 print(examples.shape)
 fig = plt.figure()
grid = convert_to_grid(examples)
 plt.imshow(grid.astype('uint8'), cmap='gray')
 plt.axis('off')
 plt.gcf().set_size_inches(7, 7)
 plt.title('Some examples of training data', fontsize=24)
 plt.show()
plt.close()
fig.savefig('training_examples.png')
 plt.close() 
<load_from_csv>
for _ in range(5):
print(swap_words('The sun is shining today, this makes me feel so good !')[0] )
x_test = pd.read_csv(".. /input/test.csv")
 x_test.head() 
<train_model>
def new_row(row, n_samples=1):
text, selected_text, textID = row['text'], row['selected_text'], row['textID']
oth_text = text.replace(selected_text, " _________________________________ ")
 new_selected_text = [get_synonyms(word)for word in selected_text.split() ]
new_oth_text = [get_synonyms(word)for word in oth_text.split() ]
new_sentences = [row]
for i in range(n_samples):
oth_text_ = " ".join([np.random.choice(l_syn, p=p, replace=True)for l_syn, p in new_oth_text])
 selected_text_ = " ".join([np.random.choice(l_syn, p=p, replace=True)for l_syn, p in new_selected_text])
 text_ = oth_text_.replace("_________________________________", selected_text_)
 if not selected_text_ in text_:
print(f'Original : {text} with target {selected_text}, oth_text {oth_text}
Transformed : {text_} with target {selected_text_}, oth_text {oth_text_}')
 continue
row2 = dc(row)
 row2['text'] = text_
row2['selected_text'] = selected_text_
row2['textID'] = f'new_{textID}'
new_sentences.append(row2)
 for r in dc(new_sentences):
r_ = dc(r)
 if np.random.choice([True, False]):
selected_text, boo = swap_words(r_['selected_text'])
 if boo:
r_['text'] = r_['text'].replace(r_['selected_text'], selected_text)
 r_['selected_text'] = selected_text
else:
oth_text, _ = swap_words(r_['text'].replace(r_['selected_text'], " _________________________________ "))
r_['text'] = oth_text.replace("_________________________________",r_['selected_text'])
 else:
oth_text, _ = swap_words(r_['text'].replace(r_['selected_text'], " _________________________________ "))
r_['text'] = oth_text.replace("_________________________________", r_['selected_text'])
 r_['textID'] = f'new_{textID}'
new_sentences.append(r_)
 new_rows = pd.concat(new_sentences, axis=1 ).transpose().drop_duplicates(subset=['text'], inplace=False, ignore_index=True)
 new_rows = new_rows.loc[new_rows['text'].apply(len)<150]
counter = 0
for i, row in new_rows.iterrows() :
if row['textID'][:4] == 'new_':
row['textID'] = row['textID']+f'_{counter}'
counter += 1
return new_rows
def pre_process_mnist(x_train, y_train, x_test):
x_train = x_train / 255.0
x_test = x_test / 255.0
batch_mask = list(range(41000, 42000))
x_validation = x_train[batch_mask]
y_validation = y_train[batch_mask]
batch_mask = list(range(41000))
x_train = x_train[batch_mask]
y_train = y_train[batch_mask]
mean_image = np.mean(x_train, axis=0)
 std = np.std(x_train, axis=0)
 for j in range(28):
for i in range(28):
if std[i, j, 0] == 0:
std[i, j, 0] = 1.0
x_train -= mean_image
x_validation -= mean_image
x_test -= mean_image
x_train /= std
x_validation /= std
x_test /= std
y_train = to_categorical(y_train, num_classes=10)
 y_validation = to_categorical(y_validation, num_classes=10)
 d_processed = {'x_train': x_train, 'y_train': y_train,
'x_validation': x_validation, 'y_validation': y_validation,
'x_test': x_test}
return d_processed
data = pre_process_mnist(x_train, y_train, x_test)
 for i, j in data.items() :
print(i + ':', j.shape)
<choose_model_class>
temp = [new_row(row, n_samples=2)for _, row in train.iterrows() ]
augmented_data = pd.concat(temp, axis=0)
 train['number'] = [t.shape[0] for t in temp]
train['number'] = train['number'].cumsum()
del temp
gc.collect()
augmented_data.drop_duplicates(subset=['text'], inplace=False, ignore_index=True)
 augmented_data.reset_index(drop=True, inplace=True)
 match_index = dc(train['number'])
 train.drop(columns='number', inplace=True)
 match_index = [0] + match_index.values.tolist()
match_borders = list(zip(match_index[:-1], match_index[1:]))
del match_index
gc.collect()
train['brackets'] = match_borders
model = Sequential()
model.add(Conv2D(64, kernel_size=7, padding='same', activation='relu', input_shape=(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=9, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=7, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size=5, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(128, kernel_size=7, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(128, kernel_size=5, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(256, kernel_size=3, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(256, kernel_size=5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(256, kernel_size=3, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Flatten())
 model.add(Dense(512, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
<train_model>
augmented_data.to_csv('extended_train.csv', index=False)
 train = augmented_data
del augmented_data
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 **(x + epochs))
epochs = 50
h = model.fit(data['x_train'], data['y_train'], batch_size=100, epochs = epochs,
validation_data =(data['x_validation'], data['y_validation']), callbacks=[annealer], verbose=1)
<train_model>
train = train.loc[train.text_len<150]
train.drop(columns=["text_len"], inplace=True)
 train.reset_index(drop=True, inplace=True )
print("Epochs={0:d}, Train accuracy={1:.5f}, Validation accuracy={2:.5f}".format(epochs, max(h.history['acc']),
max(h.history['val_acc'])))
<save_model>
train.to_csv('extended_train.csv', index=False )
model.save('my_model.h5' )<save_to_csv>
ct = augmented_data.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k, row in augmented_data.iterrows() :
text1 = " "+" ".join(row['text'].split())
 text2 = " ".join(row['selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ':
chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []
idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0:
toks.append(i)
 s_tok = sentiment_id[row['sentiment']]
input_ids[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens[k,toks[0]+2] = 1
end_tokens[k,toks[-1]+2] = 1
results = model.predict(data['x_test'])
 results = np.argmax(results, axis=1)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['Label'] = results
submission.to_csv('sample_submission.csv', index=None)
<import_modules>
ct_train = train.shape[0]
input_ids_train = np.ones(( ct_train,MAX_LEN),dtype='int32')
 attention_mask_train = np.zeros(( ct_train,MAX_LEN),dtype='int32')
 token_type_ids_train = np.zeros(( ct_train,MAX_LEN),dtype='int32')
 start_tokens_train = np.zeros(( ct_train,MAX_LEN),dtype='int32')
 end_tokens_train = np.zeros(( ct_train,MAX_LEN),dtype='int32')
 for k, row in train.iterrows() :
text1 = " "+" ".join(row['text'].split())
 text2 = " ".join(row['selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ':
chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []
idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0:
toks.append(i)
 s_tok = sentiment_id[row['sentiment']]
input_ids_train[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_train[k,:len(enc.ids)+3] = 1
if len(toks)>0:
start_tokens_train[k,toks[0]+2] = 1
end_tokens_train[k,toks[-1]+2] = 1
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from keras import layers
from keras import models
from keras import optimizers
from keras.utils import to_categorical
from sklearn.utils import shuffle
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split
from skimage import img_as_ubyte
from skimage.transform import resize
from scipy.ndimage.interpolation import rotate<load_from_csv>
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' ).fillna('' )
train_data = pd.read_csv('/kaggle/input/train.csv')
 test_data = pd.read_csv('/kaggle/input/test.csv' )<train_model>
def test_new_row(row, n_samples=1):
text, textID = row['text'], row['textID']
new_text = [get_synonyms(word)for word in text.split() ]
new_sentences = [row]
for i in range(n_samples):
text_ = " ".join([np.random.choice(l_syn, p=p, replace=True)for l_syn, p in new_text])
 row2 = dc(row)
 row2['text'] = text_
row2['textID'] = f'new_{textID}'
new_sentences.append(row2)
 for r in dc(new_sentences):
r_ = dc(r)
 text, boo = swap_words(r_['text'])
 if boo:
r_['text'] = text
r_['textID'] = f'new_{textID}'
new_sentences.append(r_)
 new_rows = pd.concat(new_sentences, axis=1 ).transpose().drop_duplicates(subset=['text'], inplace=False, ignore_index=True)
 new_rows = new_rows.loc[new_rows['text'].apply(len)<150]
counter = 0
for i, row in new_rows.iterrows() :
if row['textID'][:4] == 'new_':
row['textID'] = row['textID']+f'_{counter}'
counter += 1
return new_rows
X_model = train_data.drop('label', axis=1)
 y_model = train_data['label'].copy()
Y_finish = test_data
print('size train data:', X_model.shape)
 print('size train labels:', y_model.shape)
 print('size finish test data:', Y_finish.shape )<normalization>
temp = [test_new_row(row, n_samples=2)for _, row in test.iterrows() ]
test_augmented_data = pd.concat(temp, axis=0)
 test['number'] = [t.shape[0] for t in temp]
test['number'] = test['number'].cumsum()
del temp
gc.collect()
test_augmented_data.drop_duplicates(subset=['text'], inplace=False, ignore_index=True)
 test_augmented_data.reset_index(drop=True, inplace=True)
 test_match_index = dc(test['number'])
 test.drop(columns='number', inplace=True)
 test_match_index = [0] + test_match_index.values.tolist()
test_match_borders = list(zip(test_match_index[:-1], test_match_index[1:]))
del test_match_index
gc.collect()
test['brackets'] = test_match_borders
def img_rotate(df_x, angle):

 change_img = np.empty([df_x.shape[0], df_x.shape[1]])
 for i, image in enumerate(df_x.values):
img = rotate(image.reshape(28, 28), angle, cval=0, reshape=False, order=0)
 change_img[i] = img.ravel()
return pd.DataFrame(data=change_img, columns=df_x.columns)
 def img_zoom(df_x, scale):

 if(scale > 0.9)or(scale < -0.9):
raise Exception('scale values must be between -0.9 and 0.9')
 if scale < 0:
change_img = np.empty([df_x.shape[0], df_x.shape[1]])
 for i, image in enumerate(df_x.values):
number_around = round(28*abs(scale)/2)
 img = image.reshape(28, 28 ).astype('float32' ).copy()
img = np.pad(img, number_around, mode='constant')
 img = resize(img,(28,28), anti_aliasing=False, mode='constant', cval=0, order=0)
 change_img[i] = img.ravel().astype('int64')
 return pd.DataFrame(data=change_img, columns=df_x.columns)
 elif scale >= 0:
change_img = np.empty([df_x.shape[0], df_x.shape[1]])
 for i, image in enumerate(df_x.values):
number_slice = round(28*abs(scale)/2)
 img = image.reshape(28, 28 ).astype('float32' ).copy()
img = img[number_slice:28-number_slice, number_slice:28-number_slice]
img = resize(img,(28,28), anti_aliasing=False, mode='constant', cval=0, order=0)
 change_img[i] = img.ravel().astype('int64')
 return pd.DataFrame(data=change_img, columns=df_x.columns)
<split>
ct = test.shape[0]
test_input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 for k, row in test.iterrows() :
text1 = " "+" ".join(row['text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[row['sentiment']]
test_input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
ct = test_augmented_data.shape[0]
input_ids_t = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids_t = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k, row in test_augmented_data.iterrows() :
text1 = " "+" ".join(row['text'].split())
 enc = tokenizer.encode(text1)
 s_tok = sentiment_id[row['sentiment']]
input_ids_t[k,:len(enc.ids)+3] = [0, s_tok] + enc.ids + [2]
attention_mask_t[k,:len(enc.ids)+3] = 1
X_train, X_test, y_train, y_test = train_test_split(X_model, y_model, test_size=0.2 )<concatenate>
def save_weights(model, dst_fn):
weights = model.get_weights()
with open(dst_fn, 'wb')as f:
pickle.dump(weights, f)
 def load_weights(model, weight_fn):
with open(weight_fn, 'rb')as f:
weights = pickle.load(f)
 model.set_weights(weights)
 return model
def loss_fn(y_true, y_pred):
ll = tf.shape(y_pred)[1]
y_true = y_true[:, :ll]
loss = tf.keras.losses.categorical_crossentropy(y_true, y_pred,
from_logits=False, label_smoothing=LABEL_SMOOTHING)
 loss = tf.reduce_mean(loss)
 return loss
def build_model() :
ids = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 att = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 tok = tf.keras.layers.Input(( MAX_LEN,), dtype=tf.int32)
 padding = tf.cast(tf.equal(ids, PAD_ID), tf.int32)
 lens = MAX_LEN - tf.reduce_sum(padding, -1)
 max_len = tf.reduce_max(lens)
 ids_ = ids[:, :max_len]
att_ = att[:, :max_len]
tok_ = tok[:, :max_len]
config = RobertaConfig.from_pretrained(PATH+'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(PATH+'pretrained-roberta-base.h5',config=config)
 x = bert_model(ids_,attention_mask=att_,token_type_ids=tok_)
 x1 = tf.keras.layers.Dropout(0.1 )(x[0])
 x1 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x1)
 x1 = tf.keras.layers.LeakyReLU()(x1)
 x1 = tf.keras.layers.Dense(1 )(x1)
 x1 = tf.keras.layers.Flatten()(x1)
 x1 = tf.keras.layers.Activation('softmax' )(x1)
 x2 = tf.keras.layers.Dropout(0.1 )(x[0])
 x2 = tf.keras.layers.Conv1D(768, 2,padding='same' )(x2)
 x2 = tf.keras.layers.LeakyReLU()(x2)
 x2 = tf.keras.layers.Dense(1 )(x2)
 x2 = tf.keras.layers.Flatten()(x2)
 x2 = tf.keras.layers.Activation('softmax' )(x2)
 model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1,x2])
 optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
 model.compile(loss=loss_fn, optimizer=optimizer)
 x1_padded = tf.pad(x1, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
x2_padded = tf.pad(x2, [[0, 0], [0, MAX_LEN - max_len]], constant_values=0.)
padded_model = tf.keras.models.Model(inputs=[ids, att, tok], outputs=[x1_padded,x2_padded])
 return model, padded_model
X_train_add = X_train.append(img_zoom(X_train, 0.2))
X_train_add = X_train_add.append(img_zoom(X_train, -0.3))
X_train_add = X_train_add.append(img_rotate(X_train, 11))
X_train_add = X_train_add.append(img_rotate(X_train, -11))
y_train_add = y_train.append(y_train)
 y_train_add = y_train_add.append(y_train)
 y_train_add = y_train_add.append(y_train)
 y_train_add = y_train_add.append(y_train)
 X_train = X_train_add.copy()
y_train = y_train_add.copy()
X_train, y_train = shuffle(X_train, y_train )<data_type_conversions>
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 if(len(a)==0)&(len(b)==0):
return 0.5
c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
X_train = X_train.values.reshape(X_train.shape[0], 28, 28 ,1)
 X_train = X_train.astype('float32')/ 255
X_test = X_test.values.reshape(X_test.shape[0], 28, 28, 1)
 X_test = X_test.astype('float32')/ 255
Y_finish = Y_finish.values.reshape(Y_finish.shape[0], 28, 28 ,1)
 Y_finish = Y_finish.astype('float32')/ 255<choose_model_class>
jac = []; VER='v0'; DISPLAY=1
oof_start = np.zeros(( input_ids.shape[0],MAX_LEN))
oof_end = np.zeros(( input_ids.shape[0],MAX_LEN))
preds_start = np.zeros(( input_ids_t.shape[0],MAX_LEN))
preds_end = np.zeros(( input_ids_t.shape[0],MAX_LEN))
skf = StratifiedKFold(n_splits=2,shuffle=True,random_state=SEED)
 for fold,(idx_T,idx_V)in enumerate(skf.split(input_ids_train,train.sentiment.values)) :
idxT = np.array([i for(a,b)in train.loc[idx_T, 'brackets'] for i in range(a, b)])
 idxV = np.array([i for(a,b)in train.loc[idx_V, 'brackets'] for i in range(a, b)])
 print('
print('
print('
K.clear_session()
model, padded_model = build_model()
inpT = [input_ids[idxT,], attention_mask[idxT,], token_type_ids[idxT,]]
targetT = [start_tokens[idxT,], end_tokens[idxT,]]
inpV = [input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]]
targetV = [start_tokens[idxV,], end_tokens[idxV,]]
shuffleV = np.int32(sorted(range(len(inpV[0])) , key=lambda k:(inpV[0][k] == PAD_ID ).sum() , reverse=True))
inpV = [arr[shuffleV] for arr in inpV]
targetV = [arr[shuffleV] for arr in targetV]
weight_fn = '%s-roberta-%i.h5'%(VER,fold)
 for epoch in range(1, EPOCHS + 1):
shuffleT = np.int32(sorted(range(len(inpT[0])) , key=lambda k:(inpT[0][k] == PAD_ID ).sum() + np.random.randint(-3, 3), reverse=True))
num_batches = math.ceil(len(shuffleT)/ BATCH_SIZE)
 batch_inds = np.random.permutation(num_batches)
 shuffleT_ = []
for batch_ind in batch_inds:
shuffleT_.append(shuffleT[batch_ind * BATCH_SIZE:(batch_ind + 1)* BATCH_SIZE])
 shuffleT = np.concatenate(shuffleT_)
 inpT = [arr[shuffleT] for arr in inpT]
targetT = [arr[shuffleT] for arr in targetT]
model.fit(inpT, targetT,
epochs=epoch, initial_epoch=epoch - 1, batch_size=BATCH_SIZE, verbose=DISPLAY, callbacks=[],
validation_data=(inpV, targetV), shuffle=False)
 save_weights(model, weight_fn)
 print('Loading model...')
 load_weights(model, weight_fn)
 print('Predicting OOF...')
 oof_start[idxV,],oof_end[idxV,] = padded_model.predict([input_ids[idxV,],attention_mask[idxV,],token_type_ids[idxV,]],verbose=DISPLAY)
 print('Predicting Test...')
 preds = padded_model.predict([input_ids_t,attention_mask_t,token_type_ids_t],verbose=DISPLAY)
 preds_start += preds[0]/skf.n_splits
preds_end += preds[1]/skf.n_splits
alls = []
for k in idxV:
a = np.argmax(oof_start[k,])
 b = np.argmax(oof_end[k,])
 if a>b:
st = augmented_data.loc[k,'text']
else:
text1 = " "+" ".join(augmented_data.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 alls.append(jaccard(st,augmented_data.loc[k,'selected_text']))
jac.append(np.mean(alls))
print('>>>> FOLD %i Jaccard ='%(fold+1),np.mean(alls))
print()
def build_model() :
model = models.Sequential()
model.add(layers.Conv2D(32,(3,3), activation='relu', input_shape=(28,28,1)))
 model.add(layers.MaxPooling2D(( 2,2)))
 model.add(layers.Dropout(0.12))
model.add(layers.Conv2D(64,(3,3), activation='relu'))
model.add(layers.MaxPooling2D(( 2,2)))
 model.add(layers.Dropout(0.12))
model.add(layers.Conv2D(64,(3,3), activation='relu'))
model.add(layers.Flatten())
 model.add(layers.Dropout(0.12))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(10, activation='softmax'))
opt = optimizers.Adam(lr=0.0015, beta_1=0.9, beta_2=0.99, epsilon=None, decay=0.0, amsgrad=False)
 model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
 return model<train_model>
print('>>>> OVERALL 3Fold CV Jaccard =',np.mean(jac))
y_train = to_categorical(y_train, 10)
 y_test = to_categorical(y_test, 10)
 cnn = build_model()
cnn.fit(X_train,
y_train,
epochs=4,
batch_size=64 )<compute_test_metric>
alls = []
for k in range(input_ids_t.shape[0]):
a = np.argmax(preds_start[k,])
 b = np.argmax(preds_end[k,])
 if a>b:
st = test_augmented_data.loc[k,'text']
else:
text1 = " "+" ".join(test_augmented_data.loc[k,'text'].split())
 enc = tokenizer.encode(text1)
 st = tokenizer.decode(enc.ids[a-2:b-1])
 alls.append(st)
 test_augmented_data['selected_text'] = alls
test_loss, test_acc = cnn.evaluate(X_test, y_test)
 test_acc<predict_on_test>
test_augmented_data = pd.read_csv('.. /input/tweet-different-thresholds/test_augmented_data.csv' )
predict_test = cnn.predict_classes(X_test)
 y_correct = np.argmax(y_test, axis=1)
 correct_idx = np.nonzero(predict_test==y_correct)
 incorrect_idx = np.nonzero(predict_test!=y_correct )<compute_test_metric>
test_ = dc(test)
 def get_submission(threshold):
test1 = dc(test_)
 for ind,(a, b)in enumerate(test1.brackets.values.tolist() [0:5]):
df = test_augmented_data.loc[np.arange(a,b),['textID', 'text', 'selected_text']]
try:
selected_text = " ".join(df.loc[df.textID.str.contains('new_')==False,'selected_text'].values[0].split())
 text = " ".join(df.loc[df.textID.str.contains('new_')==False,'text'].values[0].split())
 except:
print(test1.loc[ind,:])
 print(df)
 break
print(f'Selected_text : {selected_text}, from text {text}')
 text = text.lower().split(selected_text.lower())
 print(f'When splitting :{text}')
 text_before, text_after = text[0].split() , text[-1].split()
rows = df.loc[df.textID.str.contains('new_')]
words_dic = []
i = 0
for word in text_before:
print(f'In text_before, processing word {i} : {word}')
 words_dic.append(word)
 l, p = get_synonyms(word)
 coeff = 0.5 if(word.lower() in stop or "".join(list(filter(lambda x: x not in punct, word)))in stop)else 1
rows[f'scoring_word_{i}'] = rows['selected_text'].apply(lambda x: coeff*np.average([w in x for w in l], weights=p))
i+=1
for word in selected_text.split() :
print(f'In selected_text, processing word {i} : {word}')
 words_dic.append(word)
 l, p = get_synonyms(word)
 coeff = 0.5 if(word.lower() in stop or "".join(list(filter(lambda x: x not in punct, word)))in stop)else 1
rows[f'scoring_word_{i}'] = rows['selected_text'].apply(lambda x: 2*coeff*np.average([w in x for w in l], weights=p))
i+=1
for word in text_after:
print(f'In text_after, processing word {i} : {word}')
 words_dic.append(word)
 l, p = get_synonyms(word)
 coeff = 0.5 if(word.lower() in stop or "".join(list(filter(lambda x: x not in punct, word)))in stop)else 1
rows[f'scoring_word_{i}'] = rows['selected_text'].apply(lambda x: coeff*np.average([w in x for w in l], weights=p))
i+=1
s = " ".join([words_dic[int(f.split('_')[-1])] for f in(rows[[f'scoring_word_{j}' for j in range(i)]].mean(axis=0)>=threshold ).index.tolist() ])
 test1.loc[ind, 'selected_text'] = " ".join([words_dic[int(f.split('_')[-1])] for f in(rows[[f'scoring_word_{j}' for j in range(i)]].mean(axis=0)>=threshold ).index.tolist() ])
 return test1
get_submission(threshold=0.5 )
target_names = ["Class {}".format(i)for i in range(10)]
print(classification_report(y_correct, predict_test, target_names=target_names))<predict_on_test>
test2 = get_submission(threshold=0.3)
 test3 = get_submission(threshold = 1 )
predict = cnn.predict_classes(Y_finish )<create_dataframe>
test1.to_csv('submission_05.csv', index=False)
 test2.to_csv('submission_03.csv', index=False)
 test3.to_csv('submission_1.csv', index=False )
df_out = pd.DataFrame({'ImageId': range(1, len(predict)+1),
'Label': predict} )<save_to_csv>
test_augmented_data.to_csv('test_augmented_data.csv', index=False )
df_out.to_csv('mnist_cnn.csv', index=False, header=True )<set_options>
test[['textID','selected_text']].to_csv('submission.csv', index=False )
<define_variables><EOS> 
pd.set_option('max_colwidth', 60)
 test.sample(25 )