Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<save_to_csv>
tqdm.pandas()
Activation, Dropout, SpatialDropout1D, Embedding, Concatenate, concatenate, Reshape, Flatten,
CuDNNLSTM, CuDNNGRU, Bidirectional)
 t_start = time.time()
sample_submission["target"] = simple_avg
sample_submission.to_csv("Final_simple_avg.csv", index=False )<install_modules>
embed_size = 300
max_features = 120000
maxlen = 70
nb_features = 6
EPOCHS = 10
!pip install xgboost 
<load_from_csv>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '~', '•',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
mispell_dict = {"aren't" : "are not", "can't" : "cannot", "couldn't" : "could not", "didn't" : "did not", "doesn't" : "does not",
"don't" : "do not", "hadn't" : "had not", "hasn't" : "has not", "haven't" : "have not", "he'd" : "he would", "he'll" : "he will",
"he's" : "he is", "i'd" : "I would", "i'd" : "I had", "i'll" : "I will", "i'm" : "I am", "isn't" : "is not", "it's" : "it is",
"it'll":"it will", "i've" : "I have", "let's" : "let us", "mightn't" : "might not", "mustn't" : "must not", "shan't" : "shall not",
"she'd" : "she would", "she'll" : "she will", "she's" : "she is", "shouldn't" : "should not", "that's" : "that is", "there's" : "there is",
"they'd" : "they would", "they'll" : "they will", "they're" : "they are", "they've" : "they have", "we'd" : "we would", "we're" : "we are",
"weren't" : "were not", "we've" : "we have", "what'll" : "what will", "what're" : "what are", "what's" : "what is", "what've" : "what have",
"where's" : "where is", "who'd" : "who would", "who'll" : "who will", "who're" : "who are", "who's" : "who is", "who've" : "who have",
"won't" : "will not", "wouldn't" : "would not", "you'd" : "you would", "you'll" : "you will", "you're" : "you are", "you've" : "you have",
"'re": " are", "wasn't": "was not", "we'll":" will", "didn't": "did not", "tryin'":"trying"}
bad_words = "2 girls 1 cup, 2g1c, 4r5e, 5h1t, 5hit, a$$"
bad_words = [x.strip() for x in bad_words.split(",")]
def clean_text(x):
x = str(x)
 for word in bad_words:
x = x.replace(word, "ZZZZZZ")
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{10,}', '
x = re.sub('[0-9]{5,10}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
 def split_text(x):
x = wordninja.split(x)
 return '-'.join(x)
 def add_features(df):
df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))
df['total_length'] = df['question_text'].progress_apply(len)
 df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 for df in [train_df, test_df]:
df["question_text"] = df["question_text"].str.lower()
df["question_text"] = df["question_text"].progress_apply(lambda x: clean_text(x))
df["question_text"] = df["question_text"].progress_apply(lambda x: clean_numbers(x))
df["question_text"] = df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
df["question_text"] = df["question_text"].fillna("_
print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 _train = add_features(train_df)
 _test = add_features(test_df)
 ff = ['total_length', 'capitals', 'caps_vs_length', 'num_words', 'num_unique_words', 'words_vs_unique']
features = _train[ff].fillna(0)
 test_features = _test[ff].fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features)
 train_X, valid_X, train_y, val_y = train_test_split(
 list(zip(train_df['question_text'].values, features)) ,
train_df['target'].values,
test_size=0.2, random_state=2018 
)
 test_X = test_df["question_text"].values
train_X, features = zip(*train_X)
 val_X, val_features = zip(*valid_X)
 tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 return train_X, val_X, test_X, train_y, val_y, features, val_features, test_features, tokenizer.word_index
train= pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 test= pd.read_csv('.. /input/siim-isic-melanoma-classification/test.csv')
 sub= pd.read_csv('.. /input/siim-isic-melanoma-classification/sample_submission.csv' )<data_type_conversions>
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
print(emb_mean,emb_std,"para")
 nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
train['sex'] = train['sex'].fillna('na')
 train['age_approx'] = train['age_approx'].fillna(0)
 train['anatom_site_general_challenge'] = train['anatom_site_general_challenge'].fillna('na')
 test['sex'] = test['sex'].fillna('na')
 test['age_approx'] = test['age_approx'].fillna(0)
 test['anatom_site_general_challenge'] = test['anatom_site_general_challenge'].fillna('na')
 train['sex'] = train['sex'].astype("category" ).cat.codes +1
train['anatom_site_general_challenge'] = train['anatom_site_general_challenge'].astype("category" ).cat.codes +1
train.head()<data_type_conversions>
def model_cnn(embedding_matrix):
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
test['sex'] = test['sex'].astype("category" ).cat.codes +1
test['anatom_site_general_challenge'] = test['anatom_site_general_challenge'].astype("category" ).cat.codes +1
test.head()<prepare_x_and_y>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
x_train = train[['sex', 'age_approx','anatom_site_general_challenge']]
y_train = train['target']
x_test = test[['sex', 'age_approx','anatom_site_general_challenge']]
train_DMatrix = xgb.DMatrix(x_train, label= y_train)
 test_DMatrix = xgb.DMatrix(x_test )<train_model>
def model_lstm_atten(embedding_matrix):
inp1 = Input(shape=(maxlen,), name='inp1')
 inp2 = Input(shape=(nb_features,), name='inp2')
 x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp1)
 x = SpatialDropout1D(0.25 )(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = concatenate([x, inp2])
 x = Dense(128, activation="relu" )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[inp1, inp2], outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
clf = xgb.XGBClassifier(n_estimators=2600,
max_depth=15,
objective='multi:softprob',
seed=0,
nthread=-1,
learning_rate=0.15,
num_class = 2,
scale_pos_weight =(32542/584))
clf.fit(x_train, y_train )<predict_on_test>
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit({'inp1': train_X, 'inp2': features}, train_y, batch_size=512, epochs=1,
validation_data=({'inp1': val_X, 'inp2': val_features}, val_y))
pred_val_y = model.predict({'inp1': val_X, 'inp2': val_features}, batch_size=1024, verbose=0)
 pred_test_y = model.predict({'inp1': test_X, 'inp2': test_features}, batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y
clf.predict_proba(x_test)[:,1]
sub.target = clf.predict_proba(x_test)[:,1]
sub_tabular = sub.copy()<save_to_csv>
train_X, val_X, test_X, train_y, val_y, features, val_features, test_features, word_index = load_and_prec()
sub_tabular.to_csv('submission_tabular.csv', index = False )<load_from_csv>
np.save("train_X",train_X)
 np.save("val_X",val_X)
 np.save("test_X",test_X)
 np.save("train_y",train_y)
 np.save("val_y",val_y)
 np.save("features",features)
 np.save("val_features",val_features)
 np.save("test_features",test_features)
 np.save("word_index.npy",word_index)
 train_X = np.load("train_X.npy")
 val_X = np.load("val_X.npy")
 test_X = np.load("test_X.npy")
 train_y = np.load("train_y.npy")
 val_y = np.load("val_y.npy")
 features = np.load("features.npy")
 val_features = np.load("val_features.npy")
 test_features = np.load("test_features.npy")
 word_index = np.load("word_index.npy" ).item()
sub_new = pd.read_csv(".. /input/siimisic-melanoma-subs/submission_b_Fork-of-ensemble_melanoma-ac9964_AjayKumar_v12_LB0p9596.csv")
 sub_mean = pd.read_csv(".. /input/siimisic-melanoma-subs/Submissions_MeanOf_27Models_LB0pt9571.csv")
 sub_tuned = pd.read_csv(".. /input/simple-public-blender-0-930/submission.csv")
 sub.target = sub_mean.target *0.2 + sub_new.target *0.5 + sub_tabular.target *0.2 + sub_tuned.target *0.1 
<save_to_csv>
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_2 = load_fasttext(word_index)
 embedding_matrix_3 = load_para(word_index )
sub.to_csv('ensemble_submission.csv', index = False)
 sub.head(20 )<load_from_csv>
embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2, embedding_matrix_3], axis = 0)
 np.shape(embedding_matrix )
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 print('Examples WITH Melanoma')
 imgs = train.loc[train.target==1].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
print('Examples WITHOUT Melanoma')
 imgs = train.loc[train.target==0].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()<install_modules>
outputs = []
!pip install -q efficientnet >> /dev/null<import_modules>
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix), epochs = EPOCHS)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention'] )
!pip install tensorflow-addons=='0.9.1' 
<define_search_space>
score = 0
thresh =.5
for i in np.arange(0.1, 0.991, 0.01):
y_val =(np.array(pred_val_y)> i ).astype(np.int)
 temp_score = f1_score(val_y, y_val)
 if(temp_score > score):
score = temp_score
thresh = i
print("CV: {}, Threshold: {}".format(score, thresh))
DEVICE = "TPU"
SEED = 47
FOLDS = 5
IMG_SIZES = [512,512,512,512,512]
INC2019 = [0,0,0,0,0]
INC2018 = [1,1,1,1,1]
PSEUDO = [1,1,1,1,1]
BATCH_SIZES = [32]*FOLDS
EPOCHS = [17]*FOLDS
EFF_NETS = [4,4,4,4,4]
focal_loss = 0
WGTS = [1/FOLDS]*FOLDS
TTA = 11<choose_model_class>
pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs)) ], axis = 0)
 pred_test_y =(pred_test_y > thresh ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<set_options>
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )
<feature_engineering><EOS> 
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 _train = add_features(train_df)
 _test = add_features(test_df)
 features = _train.fillna(0)
 test_features = _test.fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features)