Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<define_variables>
print(os.listdir('/kaggle/input'))
print(os.listdir('/kaggle/input/inceptionv3/'))
t0 = time.time()
vocab_original = build_vocab(pd.concat([df_train['question_text'], df_test['question_text']]))
timeSince(t0 )<categorify>
SEED = 7
np.random.seed(SEED)
 set_random_seed(SEED)
 dir_path = "/kaggle/input/"
IMG_DIM = 299
BATCH_SIZE = 8
CHANNEL_SIZE = 3
NUM_EPOCHS = 60
TRAIN_DIR = 'train_images'
TEST_DIR = 'test_images'
FREEZE_LAYERS = 2
CLASSS = {0: "No DR", 1: "Mild", 2: "Moderate", 3: "Severe", 4: "Proliferative DR"}
NUM_CLASSS = 5
t0 = time.time()
def load_embed(file):

 def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 if file.split('/')[-1] == 'wiki-news-300d-1M.vec':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file)if len(o)> 100)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index
def check_coverage(vocab, embeddings_index):

 known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('{} iv words, {} unique'.format(nb_known_words, len(known_words)))
 print('{} oov words, {} unique'.format(nb_unknown_words, len(unknown_words)))
 print('Found embeddings for {:.3%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.3%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
def create_emb_matrix(nb_words, embed_size):

 return np.zeros(( nb_words, embed_size), dtype=np.float32)
 def fill_emb_matrix(word_idx, emb_matrix, emb_index):

 for word, i in word_idx:
emb_vector = emb_index.get(word)
 if emb_vector is not None:
emb_matrix[i] = emb_vector
return emb_matrix
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding")
 timeSince(t0 )<load_pretrained>
ROOT_PATH = '/kaggle/input/aptos2019-blindness-detection'
TRAIN_PATH = '/kaggle/input/aptos2019-blindness-detection/' + TRAIN_DIR
TEST_PATH = '/kaggle/input/aptos2019-blindness-detection/' + TEST_DIR
dir_path = ROOT_PATH + '/'
t0 = time.time()
conc_embedding = None
word_index = word_to_idx
nb_words = min(hparam['VOCAB_SIZE'], len(word_index)+ 1)
 hparam['VOCAB_SIZE'] = nb_words
print(hparam['VOCAB_SIZE'], len(word_index)+ 1)
 print(f"Got a vocab size of {nb_words} number of words")
 for embedding in embeddings:
emb_name = embedding['name']
emb_path = embedding['path']
print("Running procedure on {}".format(emb_name))
print("Loading {}".format(emb_name))
emb_index = load_embed(emb_path)
 print("Adding lowercase to {}".format(emb_name))
add_lower(emb_index, vocab_original)
 _ = check_coverage(final_vocab, emb_index)
 emb_size = 300
hparam['WORD_EMB_DIM'] += emb_size
emb_matrix = create_emb_matrix(nb_words, emb_size)
 print(emb_matrix.size)
 print(emb_matrix.shape)
 emb_matrix = fill_emb_matrix(word_index.items() , emb_matrix, emb_index)
 if conc_embedding is not None:
conc_embedding = np.concatenate(( conc_embedding, emb_matrix), axis=1)
 print("concatenated! now got shape: {}".format(conc_embedding.shape))
else:
conc_embedding = emb_matrix
del emb_matrix, emb_index, emb_name, emb_path, emb_size
timeSince(t0 )<categorify>
train_img_names = glob.glob(TRAIN_PATH + '/*.png')
 df_train = pd.read_csv(ROOT_PATH + '/train.csv')
t0 = time.time()
def embed_word_to_int(X, vocab_to_int):
embedded_X = []
for q in X:
tmp_X = []
for w in q.split() :
tmp_X.append(vocab_to_int[w])
 embedded_X.append(tmp_X)
 return embedded_X
X_train = embed_word_to_int(df_train['processed_questions'].values, word_to_idx)
 X_test = embed_word_to_int(df_test['processed_questions'].values, word_to_idx)
 pad_length = hparam['PAD_LENGTH']
X_train_pad = pad_sequences(X_train, maxlen=pad_length, padding='pre', truncating='pre')
 X_test_pad = pad_sequences(X_test, maxlen=pad_length, padding='pre', truncating='pre')
 print(df_train['processed_questions'][3333])
 print(X_train[3333])
 print(X_train_pad[3333])
 timeSince(t0 )<choose_model_class>
test_img_names = glob.glob(TEST_PATH + '/*.png')
 df_test = pd.read_csv(ROOT_PATH + '/test.csv')
t0 = time.time()
kfold = StratifiedKFold(n_splits=hparam['KFOLDS'], shuffle=True, random_state=2019)
 timeSince(t0 )<train_model>
gc.collect()
t0 = time.time()
X = X_train_pad
y = df_train['target'].values
results = []
for fold,(train_index, val_index)in enumerate(kfold.split(X, y)) :
X_train, X_val = X[train_index], X[val_index]
y_train, y_val = y[train_index], y[val_index]
print(f"Training on {len(X_train)} and validating on {len(X_val)} number of words")
 dataset = {'X_train': X_train, 'y_train': y_train,
'X_val': X_val, 'y_val': y_val,
'X_test': X_test_pad}
test_preds, val_preds, thresh, f1 = train_val_pred(dataset, hparam, conc_embedding)
 print("len(test_preds)= {}, len(val_preds)= {}, thresh = {} at f1 = {}".format(len(test_preds),
len(val_preds),
thresh,
f1))
new_result = {'name': 'fold-' + str(fold),
'test_preds': test_preds,
'val_preds': val_preds,
'thresh': thresh,
'f1': f1}
results.append(new_result)
 timeSince(t0 )<define_variables>
x_train, x_test, y_train, y_test = train_test_split(df_train.id_code, df_train.diagnosis, test_size=0.2,
random_state=SEED, stratify=df_train.diagnosis )
t0 = time.time()
print("Got {} number of results!".format(len(results)))
 avg_thresh = 0
for result in results:
print("{} gave f1 score {} with thresh {}".format(result['name'], result['f1'], result['thresh']))
avg_thresh += result['thresh']
avg_thresh = avg_thresh / len(results)
 print("Got an average threshold at {}".format(avg_thresh))
timeSince(t0 )<categorify>
input_tensor = Input(shape =(299, 299, 3))
base_model = InceptionV3(weights=None, include_top=False, input_tensor=input_tensor)
 base_model.load_weights('/kaggle/input/inceptionv3/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5')
 x = base_model.output
output = BatchNormalization()(x)
 x = GlobalAveragePooling2D()(x)
 x = Dense(1024, activation='relu' )(x)
 predictions = Dense(NUM_CLASSS, activation='softmax' )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 for layer in base_model.layers:
layer.trainable = False
for layer in model.layers:
layer.trainable = True
print(model.summary() )
t0 = time.time()
print("Using treshold {}".format(avg_thresh))
factor = 1.0 / len(results)
 pred_test_y = results[0]['test_preds'] * factor
print("Using factor: ", factor)
 for i in range(1, len(results)) :
pred_test_y += factor * results[i]['test_preds']
pred_test_y_res =(pred_test_y > avg_thresh ).astype(int)
 results_dict = {'qid':df_test['qid'].values, 'prediction':[]}
for prediction in pred_test_y_res:
results_dict['prediction'].append(prediction[0])
 print(results_dict['qid'][:15])
 print(results_dict['prediction'][:15])
 df = pd.DataFrame(data=results_dict)
 df.to_csv('submission.csv', index=False)
 print("Saved csv to disk!")
 timeSince(t0 )<define_variables>
epochs = 25
lrate = 0.01
decay = lrate/epochs
sgd = SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=False)
 model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
<define_variables>
print("available RAM:", psutil.virtual_memory())
 gc.collect()
print("available RAM:", psutil.virtual_memory())
 df_train.id_code = df_train.id_code.apply(lambda x: x + ".png")
 df_test.id_code = df_test.id_code.apply(lambda x: x + ".png")
 df_train['diagnosis'] = df_train['diagnosis'].astype('str' )
embed_size = 300
max_features = 95000
maxlen = 70<import_modules>
train_datagen = image.ImageDataGenerator(rescale=1./ 255, validation_split=0.15, horizontal_flip=True,
vertical_flip=True, rotation_range=360, zoom_range=0.2, shear_range=0.1 )
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import re
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D,BatchNormalization
from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.initializers import glorot_normal, orthogonal
import tensorflow as tf<define_variables>
train_generator = train_datagen.flow_from_dataframe(dataframe=df_train,
directory= TRAIN_PATH + '/',
x_col='id_code',
y_col='diagnosis',
batch_size=BATCH_SIZE,
class_mode='categorical',
target_size=(IMG_DIM, IMG_DIM),
subset='training',
shaffle=True,
seed=SEED 
)
 valid_generator = train_datagen.flow_from_dataframe(dataframe=df_train,
directory= TRAIN_PATH + '/',
x_col='id_code',
y_col='diagnosis',
batch_size=BATCH_SIZE,
class_mode='categorical',
target_size=(IMG_DIM, IMG_DIM),
subset='validation',
shaffle=True,
seed=SEED 
)
 gc.collect()
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x<load_from_csv>
NUB_TRAIN_STEPS = train_generator.n // train_generator.batch_size
NUB_VALID_STEPS = valid_generator.n // valid_generator.batch_size
NUB_TRAIN_STEPS, NUB_VALID_STEPS
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 train_df["question_text"] = train_df["question_text"].apply(lambda x: x.lower())
 test_df["question_text"] = test_df["question_text"].apply(lambda x: x.lower())
 train_df["question_text"] = train_df["question_text"].apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df, val_df = train_test_split(train_df, test_size=0.04, random_state=2018)
 train_X = train_df["question_text"].fillna("_
val_X = val_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]
return train_X, val_X, test_X, train_y, val_y, tokenizer.word_index<statistical_test>
eraly_stop = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=3, verbose=1, mode='auto')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', min_delta=0.0004, patience=2, factor=0.1, min_lr=1e-6, mode='auto',
verbose=1 )
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
print(emb_mean,emb_std,"para")
 nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<choose_model_class>
history = model.fit_generator(generator=train_generator,
steps_per_epoch=NUB_TRAIN_STEPS,
validation_data=valid_generator,
validation_steps=NUB_VALID_STEPS,
epochs=NUM_EPOCHS,
callbacks=[eraly_stop, reduce_lr],
verbose=1)
 gc.collect()
def model_cnn(embedding_matrix):
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<normalization>
test_datagen = image.ImageDataGenerator(rescale=1./ 255, validation_split=0.2, horizontal_flip=True)
 test_generator = test_datagen.flow_from_dataframe(dataframe=df_test,
directory= TEST_PATH + '/',
x_col="id_code",
target_size=(IMG_DIM, IMG_DIM),
batch_size=1,
shuffle=False,
class_mode=None,
seed=SEED)
 print(df_test.shape[0])
 gc.collect()
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )<choose_model_class>
tta_steps = 5
preds_tta = []
for i in tqdm(range(tta_steps)) :
test_generator.reset()
preds = model.predict_generator(generator=test_generator, steps=ceil(df_test.shape[0]))
preds_tta.append(preds)
def capsule(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.2 )(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True,
kernel_initializer=glorot_normal(seed=12300), recurrent_initializer=orthogonal(gain=1.0, seed=10000)) )(x)
 x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True )(x)
 x = Flatten()(x)
 x = Dense(100, activation="relu", kernel_initializer=glorot_normal(seed=12300))(x)
 x = Dropout(0.12 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model<choose_model_class>
final_pred = np.mean(preds_tta, axis=0)
 predicted_class_indices = np.argmax(final_pred, axis=1)
 len(predicted_class_indices )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
results = pd.DataFrame({"id_code": test_generator.filenames, "diagnosis": predicted_class_indices})
 results.id_code = results.id_code.apply(lambda x: x[:-4])
 results.to_csv("submission.csv", index=False )