Kaggle competition: COVID19 Global Forecasting (Week 4)
def model_cnn(embedding_matrix):
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
%matplotlib inline
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
df_train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/train.csv")
 df_test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/test.csv")
 df_submission = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/submission.csv" )
def model_lstm_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
df_train['Date'] = pd.to_datetime(df_train['Date'])
 df_test['Date'] = pd.to_datetime(df_test['Date'])
def model_gru_srk_atten(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
Global_Top_Cases = df_train[df_train['Date'] == df_train['Date'].max() ].groupby(['Country_Region','Date'] )
def model_lstm_du(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 conc = Dense(64, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
G_cases = Global_Top_Cases.sum().sort_values(['ConfirmedCases'], ascending = False )
def model_gru_atten_3(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True))(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<predict_on_test>
G1 = G_cases.drop(['Id'], axis = 1 )
def train_pred(model, epochs=2):
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))
pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y<define_variables>
G1['ConfirmedCases_percent_change'] = G1['ConfirmedCases'].pct_change()
G1['Fatalities_percent_change'] = G1['Fatalities'].pct_change()
train_X, val_X, test_X, train_y, val_y, word_index = load_and_prec()
vocab = []
for w,k in word_index.items() :
vocab.append(w)
 if k >= max_features:
break<categorify>
G2 = G1.drop(['ConfirmedCases', 'Fatalities'], axis = 1 )
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index )<define_variables>
us = df_train[df_train['Country_Region'] == 'US'].groupby(['Province_State'] ).max()
outputs = []<predict_on_test>
us_cases = us.sort_values(['ConfirmedCases'], ascending = False )
pred_val_y, pred_test_y = train_pred(model_gru_atten_3(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '3 GRU w/ atten'] )<predict_on_test>
us_cases = us_cases.drop(['Id', 'Date','Country_Region'], axis = 1 )
pred_val_y, pred_test_y = train_pred(model_cnn(embedding_matrix_1), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, '2d CNN GloVe'] )<predict_on_test>
us_cases['ConfirmedCases_percent_change'] = us_cases['ConfirmedCases'].pct_change()
us_cases['Fatalities_percent_change'] = us_cases['Fatalities'].pct_change()
pred_val_y, pred_test_y = train_pred(model_lstm_du(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, 'LSTM DU'] )<predict_on_test>
us_cases = us_cases.drop(['ConfirmedCases', 'Fatalities'], axis = 1 )
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention'] )<predict_on_test>
df_train['Country_Region'] = df_train['Country_Region'] + ' ' + df_train['Province_State']
df_test['Country_Region'] = df_test['Country_Region'] + ' ' + df_test['Province_State']
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix_1), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention GloVe'] )<predict_on_test>
df_train['Date'] = list(df_train['Date'].dt.strftime('%Y,%m,%d'))
df_test['Date'] = list(df_test['Date'].dt.strftime('%Y,%m,%d'))
pred_val_y, pred_test_y = train_pred(model_lstm_atten(embedding_matrix_3), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, '2 LSTM w/ attention Para'] )<save_to_csv>
def createDateFields(df):
year = []
month = []
day = []
for i in df['Date']:
i = i.split(",")
 year.append(i[0])
 month.append(i[1])
 day.append(i[2])
 df['year'] = year
df['month'] = month
df['day'] = day
return df
coefs = [0.20, 0.079, 0.12, 0.14, 0.15, 0.17, 0.14]
pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs)) ], axis = 0)
 pred_test_y =(pred_test_y > 0.34 ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<train_model>
df_train = createDateFields(df_train)
 df_test = createDateFields(df_test)
t_finish = time.time()
print(f"Kernel run time = {(t_finish-t_start)/3600} hours" )<import_modules>
df_train['ConfirmedCases']= df_train['ConfirmedCases'].apply(int)
 df_train['Fatalities'] = df_train['Fatalities'].apply(int )
tqdm.pandas() 
<define_variables>
y1 = df_train.ConfirmedCases
y2 = df_train.Fatalities
del df_train['ConfirmedCases']
del df_train['Fatalities']
embed_size = 300
max_features = 95000
maxlen = 70
TOP_N_FIRST_WORDS = 40
NUM_CUSTOM_FEATURES = 15 + TOP_N_FIRST_WORDS
NUM_TOTAL_FEATURES = 19<feature_engineering>
from sklearn.preprocessing import LabelEncoder
glove_EMBEDDING_FILE = open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 para_embedding_file = open('.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt', encoding="utf8", errors='ignore')
 def load_glove(word_index):
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in glove_EMBEDDING_FILE)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in para_embedding_file if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<categorify>
lable_encoder = LabelEncoder()
df_train['Country_Region'] = lable_encoder.fit_transform(df_train['Country_Region'])
 df_test['Country_Region'] = lable_encoder.transform(df_test['Country_Region'] )
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
x = re.sub('[0-9]k', '
return x<string_transform>
features = ['Country_Region', 'month', 'day']
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispell_dict = {'colour':'color',
'grey' : 'gray',
'recognised' : 'recognized',
'recognise' : 'recognize',
'defence' : 'defense',
'programmes' : 'programme',
'centre' : 'center',
'didnt' : 'did not',
'doesnt' : 'does not',
'isnt' : 'is not',
'Isnt' : 'is not',
'hasnt' : 'has not',
'wasnt' : 'was not',
'Doesnt' : 'does not',
'Shouldnt' : 'should not',
'shouldnt' : 'should not',
'favourite' : 'favorite',
'travelling' : 'traveling',
'counselling' : 'counseling',
'theatre' : 'theater',
'cancelled' : 'canceled',
'realized' : 'realised',
'memorise' : 'memorize',
'labour' : 'labor',
'organisation':'organization',
'wwii':'world war 2',
'citicise':'criticize',
'wwwyoutubecom' : 'youtube',
'instagram': 'social medium',
'whatsapp': 'social medium',
'WeChat' : 'social medium',
'snapchat': 'social medium',
'Snapchat' : 'social medium',
'Pinterest' : 'social medium',
'bitcoins' : 'cryptocurrency',
'bitcoin' : 'cryptocurrency',
'Ethereum' : 'cryptocurrency',
'cryptocurrencies' : 'cryptocurrency',
'ethereum' : 'cryptocurrency',
'Coinbase' : 'cryptocurrency',
'Blockchain' : 'cryptocurrency',
'Cryptocurrency' : 'cryptocurrency',
'Litecoin' : 'cryptocurrency',
'coinbase' : 'cryptocurrency',
'altcoin' : 'cryptocurrency',
'litecoin' : 'cryptocurrency',
'cryptos' : 'cryptocurrency',
'Fortnite' : 'game',
'Nodejs' : 'programming',
'nodejs' : 'programming',
'ReactJS' : 'programming',
'Golang' : 'programming',
'counsellor' : 'counselling',
'Tensorflow' : 'machine learning',
'TensorFlow' : 'machine learning',
'DeepMind' : 'machine learning',
'Codeforces' : 'programming',
'HackerRank' : 'programming',
'CodeChef' : 'programming',
'AngularJS' : 'programming',
'PewDiePie' : 'gaming social medium',
'brexit' : 'Brexit',
'Xiaomi' : 'phone company',
'OnePlus' : 'phone company',
'mastrubation' : 'masturbation',
'colour': 'color',
'centre': 'center',
'favourite': 'favorite',
'travelling': 'traveling',
'theatre': 'theater',
'cancelled': 'canceled',
'labour': 'labor',
'organisation': 'organization',
'wwii': 'world war 2',
'citicise': 'criticize',
'youtu ': 'youtube ',
'Qoura': 'Quora',
'sallary': 'salary',
'Whta': 'What',
'narcisist': 'narcissist',
'howdo': 'how do',
'whatare': 'what are',
'howcan': 'how can',
'howmuch': 'how much',
'howmany': 'how many',
'whydo': 'why do',
'doI': 'do I',
'theBest': 'the best',
'howdoes': 'how does',
'mastrubation': 'masturbation',
'mastrubate': 'masturbate',
"mastrubating": 'masturbating',
'pennis': 'penis',
'Etherium': 'cryptocurrency',
'narcissit': 'narcissist',
'bigdata': 'big data',
'2k17': '2017',
'2k18': '2018',
'qouta': 'quota',
'exboyfriend': 'ex boyfriend',
'airhostess': 'air hostess',
"whst": 'what',
'watsapp': 'whatsapp',
'demonitisation': 'demonetization',
'demonitization': 'demonetization',
'demonetisation': 'demonetization',
'mofo': 'fuck',
'ww2': 'world war 2',
'havent': 'have not',
'neonazis': 'Nazis',
'hillary clinton': 'Hillary Clinton',
'donald trump': 'Donald Trump'
}
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
<string_transform>
X = df_train[features]
string.printable
ascii_chars = string.printable
ascii_chars += " áéíóúàèìòùâêîôûäëïöüñõç"
def contains_non_english(text):
if all(char in ascii_chars for char in text):
return 0
else:
return 1
def remove_non_english(text):
return ''.join(filter(lambda x: x in ascii_chars, text))
def get_first_word(word):
if(type(word)!= "float"):
return word.split(" ")[0]
return "-1"
def get_cap_vs_length(row):
if row["total_length"] == 0:
return -1
return float(row['capitals'])/float(row['total_length'])
 def calc_max_word_len(sentence):
maxLen = 0
for word in sentence:
maxLen = max(maxLen, len(word))
return maxLen
def remove_singles(text):
return ' '.join([w for w in text.split() if(( len(w)>1)or(w.lower() == "i")or(w.lower() == "a")) ])
 def clean_text(x):
x = str(x)
 x = x.replace("-", '')
 x = x.replace("/", '')
 x = x.replace("'",'')
 x = re.sub('\s+', ' ', x ).strip()
for punct in '&':
x = x.replace(punct, f' {punct} ')
 for punct in '?!.,"
x = x.replace(punct, '')
 return x<load_from_csv>
test_features = ['Country_Region', 'month', 'day']
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train.shape)
 print("Test shape : ",test.shape )<string_transform>
X_test = df_test[test_features]
def num_bad_words(text):
badwords = 0
input_words=text.split()
for word in input_words:
if word in bad_words:
badwords += 1
return badwords
def num_good_words(text):
goodwords = 0
input_words=text.split()
for word in input_words:
if word in good_words:
goodwords += 1
return goodwords<feature_engineering>
from sklearn.preprocessing import MinMaxScaler
print("--- Generating non_eng")
 train["non_eng"] = train["question_text"].map(lambda x: contains_non_english(x))
test["non_eng"] = test["question_text"].map(lambda x: contains_non_english(x))
print("--- Generating first_word")
 train["first_word"] = train["question_text"].map(lambda x: get_first_word(x))
test["first_word"] = test["question_text"].map(lambda x: get_first_word(x))
print("--- Generating total_length(num chars)")
 train['total_length'] = train['question_text'].apply(len)
 test['total_length'] = test['question_text'].apply(len)
 print("--- Generating capitals")
 train['capitals'] = train['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))
test['capitals'] = test['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))
print("--- Generating caps_vs_length")
 train['caps_vs_length'] = train.apply(lambda row: get_cap_vs_length(row),axis=1)
 test['caps_vs_length'] = test.apply(lambda row: get_cap_vs_length(row),axis=1)
 print("--- Generating num_question_marks")
 train['num_question_marks'] = train['question_text'].apply(lambda comment: comment.count('?'))
test['num_question_marks'] = test['question_text'].apply(lambda comment: comment.count('?'))
print("--- Generating num_punctuation")
 train['num_punctuation'] = train['question_text'].apply(lambda comment: sum(comment.count(w)for w in '.,;:'))
test['num_punctuation'] = test['question_text'].apply(lambda comment: sum(comment.count(w)for w in '.,;:'))
print("--- Generating num_words")
 train['num_words'] = train['question_text'].apply(lambda comment: len(re.sub(r'[^\w\s]','',comment ).split(" ")))
 test['num_words'] = test['question_text'].apply(lambda comment: len(re.sub(r'[^\w\s]','',comment ).split(" ")))
 print("--- Generating num_unique_words")
 train['num_unique_words'] = train['question_text'].apply(lambda comment: len(set(w for w in comment.split())))
 test['num_unique_words'] = test['question_text'].apply(lambda comment: len(set(w for w in comment.split())))
 print("--- Generating words_vs_unique")
 train['words_vs_unique'] = train['num_unique_words'] / train['num_words']
test['words_vs_unique'] = test['num_unique_words'] / test['num_words']
print("--- Generating num_sentences")
 train['num_sentences'] = train['question_text'].apply(lambda comment: len(re.split(r'[.!?]+', comment)))
 test['num_sentences'] = test['question_text'].apply(lambda comment: len(re.split(r'[.!?]+', comment)))
 print("--- Generating max_word_len")
 train['max_word_len'] = train['question_text'].apply(lambda comment: calc_max_word_len(re.sub(r'[^\w\s]','',comment ).split(" ")))
 test['max_word_len'] = test['question_text'].apply(lambda comment: calc_max_word_len(re.sub(r'[^\w\s]','',comment ).split(" ")))
 print("cleaning text")
 train["question_text"] = train["question_text"].apply(lambda x: clean_text(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_text(x))
print("remove single characters")
 train["question_text"] = train["question_text"].apply(lambda x: remove_singles(x))
test["question_text"] = test["question_text"].apply(lambda x: remove_singles(x))
print("cleaning numbers")
 train["question_text"] = train["question_text"].apply(lambda x: clean_numbers(x))
test["question_text"] = test["question_text"].apply(lambda x: clean_numbers(x))
print("cleaning misspellings")
 train["question_text"] = train["question_text"].apply(lambda x: replace_typical_misspell(x))
test["question_text"] = test["question_text"].apply(lambda x: replace_typical_misspell(x))
print("filling missing values")
 print('cleaning characters')
 train["question_text"] = train["question_text"].map(lambda x: remove_non_english(x))
test["question_text"] = test["question_text"].map(lambda x: remove_non_english(x))
train["question_text"].fillna("" ).values
test["question_text"].fillna("" ).values
train1_df = train[train["target"]==1]
train0_df = train[train["target"]==0]
def generate_ngrams(text, n_gram=1):
token = [token for token in text.lower().split(" ")if token != "" if token not in STOPWORDS]
ngrams = zip(*[token[i:] for i in range(n_gram)])
 return [" ".join(ngram)for ngram in ngrams]
freq_dict_bad = defaultdict(int)
 for sent in train1_df["question_text"]:
for word in generate_ngrams(sent):
freq_dict_bad[word] += 1
freq_dict_bad = dict(freq_dict_bad)
 freq_dict_good = defaultdict(int)
 for sent in train0_df["question_text"]:
for word in generate_ngrams(sent):
freq_dict_good[word] += 1
freq_dict_good = dict(freq_dict_good)
 bad_words = sorted(freq_dict_bad, key=freq_dict_bad.get, reverse=True)[:1000]
good_words = sorted(freq_dict_good, key=freq_dict_good.get, reverse=True)[:1000]
print("--- Generating num_bad_words")
 train["num_bad_words"] = train["question_text"].map(lambda x: num_bad_words(x))
test["num_bad_words"] = test["question_text"].map(lambda x: num_bad_words(x))
print("--- Generating num_good_words")
 train["num_good_words"] = train["question_text"].map(lambda x: num_good_words(x))
test["num_good_words"] = test["question_text"].map(lambda x: num_good_words(x)) 
<categorify>
min_max_scaler = MinMaxScaler()
X = min_max_scaler.fit_transform(X.values)
 X_test = min_max_scaler.transform(X_test.values )
top_x = ['What', 'How', 'Why', 'Is', 'Can', 'Which', 'Do', 'If', 'Are', 'Who', 'I', 'Does', 'Where', 'Should', 'When', 'Will', "What's", 'In', 'Would', 'Have', 'Did', 'My', 'Has', 'As', 'Could', "I'm", 'Was', 'A', 'The', 'What’s', 'For', 'After', 'At', 'With', 'Am', 'Were', 'From', 'Since', 'To', 'On']
train["first_word"] = train["first_word"].map(lambda x: x if x in top_x else "Other")
 test["first_word"] = test["first_word"].map(lambda x: x if x in top_x else "Other")
 one_hot_encoded_first_word_train = pd.get_dummies(train["first_word"])
 one_hot_encoded_first_word_test = pd.get_dummies(test["first_word"])
 original_headers_train = list(train.columns.values)
 original_headers_test = list(test.columns.values)
 one_hot_encoded_first_word_headers_train = one_hot_encoded_first_word_train.columns.values
one_hot_encoded_first_word_headers_test = one_hot_encoded_first_word_test.columns.values
one_hot_encoded_first_word_headers_train = ["first_word_" + x for x in one_hot_encoded_first_word_headers_train]
one_hot_encoded_first_word_headers_test = ["first_word_" + x for x in one_hot_encoded_first_word_headers_test]
new_train_headers = original_headers_train + one_hot_encoded_first_word_headers_train
new_test_headers = original_headers_test + one_hot_encoded_first_word_headers_test
train_with_features = pd.concat([train, one_hot_encoded_first_word_test], axis=1, ignore_index=True)
 train_with_features.columns = new_train_headers
test_with_features = pd.concat([test, one_hot_encoded_first_word_test], axis=1, ignore_index=True)
 test_with_features.columns = new_test_headers<define_variables>
import xgboost as xgb
to_drop = [
'first_word_Other', 'first_word_Can', 'first_word_Where',
'first_word_Since', 'first_word_Did',
'first_word_If', "first_word_What's", 'first_word_Should',
'first_word_Who', 'first_word_I', 'first_word_Was',
'first_word_The', 'first_word_When', 'first_word_Is',
'first_word_Would', 'first_word_In', 'first_word_As',
'first_word_Were', 'first_word_Will', 'first_word_What’s',
'first_word_My', 'first_word_With', 'first_word_Am',
'first_word_To', 'first_word_At', 'first_word_From',
'first_word_Has', 'first_word_After',
'first_word_Does', 'first_word_Could', 'first_word_A',
"first_word_I'm", 'first_word_For', 'first_word_Have',
'first_word_On']
train_with_features = train_with_features.drop(to_drop,axis=1)
 test_with_features = test_with_features.drop(to_drop,axis=1)
<prepare_x_and_y>
xg_reg_model1 = xgb.XGBRegressor(n_estimators = 1500,random_state = 0, max_depth = 15, learning_rate = 0.1 )
def tokenize() :
train_X = train["question_text"]
test_X = test["question_text"]
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train['target'].values
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
return train_X, test_X, train_y, tokenizer.word_index 
<normalization>
xg_reg_model1.fit(X,y1);
train_X, test_X, train_y, word_index = tokenize()
embedding_matrix_1 = load_glove(word_index)
 embedding_matrix_3 = load_para(word_index)
 embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)
 np.shape(embedding_matrix )<concatenate>
y_pred1 = xg_reg_model1.predict(X_test)
df = pd.concat([train_with_features.drop(['target'], axis=1), test], sort = 'False')
 print(df.shape)
<normalization>
xg_reg_model2 = xgb.XGBRegressor(n_estimators = 1500,random_state =0, max_depth = 15, learning_rate = 0.1)
 xg_reg_model2.fit(X,y2);
features_names = df.drop(["qid","question_text","first_word"],axis=1 ).columns.values
tmp_df = df[features_names]
scaler = MinMaxScaler()
tmp_df = scaler.fit_transform(tmp_df)
 train_data = tmp_df[0 : train.shape[0]]
test_data = tmp_df[train.shape[0] :(train.shape[0] + test.shape[0])]
x_features_train = train_data
x_features_test = test_data
pd.DataFrame(x_features_train ).head() 
<choose_model_class>
y_pred2 = xg_reg_model2.predict(X_test)
def model_gru_cap(embedding_matrix):
input_embeddings = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(input_embeddings)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True,
kernel_initializer=glorot_normal(seed=12300), recurrent_initializer=orthogonal(gain=1.0, seed=10000)) )(x)
 x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True )(x)
 x = Flatten()(x)

input_features = Input(shape =(NUM_TOTAL_FEATURES,))
x = Dense(100, activation="relu", kernel_initializer=glorot_normal(seed=12300))(x)
 x = Dropout(0.12 )(x)
 x = BatchNormalization()(x)
 outp = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[input_embeddings, input_features], outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1])
 return model<compute_train_metric>
y_pred1 , y_pred2 = np.round(y_pred1),np.round(y_pred2)
 y_pred1[y_pred1 < 0] = 0
y_pred2[y_pred2 < 0] = 0
df_submission['ConfirmedCases'] = y_pred1.astype(int)
 df_submission['Fatalities'] = y_pred2.astype(int )
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<normalization>
df_submission.to_csv("submission.csv",index = False )