Kaggle competition: COVID19 Global Forecasting (Week 4)
df_test.drop(labels=["Province_State","Country_Region"], axis=1, inplace=True)
 df_test.sample(5 )<count_unique_values>
N_FEATURES = 5
print(list(df_train.Date.unique()))
print(len(df_train.Date.unique()))<count_unique_values>
countries = pd.read_csv(
 '/kaggle/input/countries-of-the-world/countries of the world.csv', decimal=',' 
)
 countries['Country'] = countries.Country.str.lower()
countries = countries.set_index('Country')
print(list(df_test.Date.unique()))
print(len(df_test.Date.unique()))<count_unique_values>
train_data = pd.read_csv(
 '/kaggle/input/covid19-global-forecasting-week-4/train.csv'
).set_index('Id')
world_list_confirmed = []
world_list_fatality = []
for date in list(df_train.Date.unique()):
confirmed = df_train.ConfirmedCases[df_train.Date == date]
world_list_confirmed.append(sum(confirmed))
fatality = df_train.Fatalities[df_train.Date == date]
world_list_fatality.append(sum(fatality))<prepare_output>
def transform_area(data):
data["Area"] = ""
data.loc[~data.Province_State.isna() , "Area"] = data.loc[~data.Province_State.isna() ].Country_Region.str.lower() + "/" + data.Province_State.str.lower()
data.loc[data.Province_State.isna() , "Area"] = data.loc[data.Province_State.isna() ].Country_Region.str.lower()
data = data.drop('Province_State', axis='columns')
 return data
def transform_countries(data):
data['Density'] = countries['Pop.Density(per sq.mi.) '].mean()
data['InfantMortality'] = countries['Infant mortality(per 1000 births)'].mean()
for i in data.index:
country = data.loc[i, "Country_Region"].lower()
if country in countries.index:
data.loc[i, "Density"] = countries.loc[country, 'Pop.Density(per sq.mi.) ']
return data
Date_submission = date_list[71:84]
Date_prediction = date_test[13:]
print(Date_submission)
 print(Date_prediction)
 print(len(Date_prediction))<split>
train_data = transform_area(train_data)
 train_data = transform_countries(train_data)
 train_data.head()
def submit_from_pred(date_list, value_list, Unique_Region, test_df, submission_df, num):
if len(date_list)!= len(value_list):
print("Error, the length of these two lists are not equal")
 else:
for i in range(len(date_list)) :
pred = value_list[i]
selected_df = test_df[(test_df["Date"] == date_list[i])&(test_df["Unique_Region"] == Unique_Region)]
forecastID = selected_df["ForecastId"].iloc[0]
if num == 0:
submission_df.ConfirmedCases[submission_df["ForecastId"] == forecastID] = pred
else:
submission_df.Fatalities[submission_df["ForecastId"] == forecastID] = pred<feature_engineering>
X_train = {}
X_test = {}
Y_train = {}
Y_test = {}
for area in list(set(train_data.Area)) :
print(area)
 area_data = train_data.loc[train_data.Area == area].set_index('Date' ).sort_index()
Y = area_data[['ConfirmedCases', 'Fatalities']] - area_data.shift(1)[['ConfirmedCases', 'Fatalities']]
dic = OrderedDict()
for i in range(1, 1 + N_FEATURES):
dic['CC_{}'.format(i)] = area_data.shift(i)['ConfirmedCases']
dic['F_{}'.format(i)] = area_data.shift(i)['Fatalities']
dic['Density'] = area_data["Density"]
dic['InfantMortality'] = area_data["InfantMortality"]
X = pd.DataFrame(dic, index=area_data.index)
 X = X.dropna(axis='index')
 Y = Y.loc[X.index]
X_train[area] = X.iloc[:int(len(X)* 0.8)]
X_test[area] = X.iloc[int(len(X)* 0.8):]
Y_train[area] = Y.iloc[:int(len(X)* 0.8)]
Y_test[area] = Y.iloc[int(len(X)* 0.8):]
def from_diff_to_var(diff_series, start_value):
value = start_value
for i in range(len(diff_series)) :
value += diff_series[i]
diff_series[i] = value
return diff_series<predict_on_test>
X_train = np.vstack(list(X_train.values()))
X_test = np.vstack(list(X_test.values()))
Y_train = np.vstack(list(Y_train.values()))
Y_test = np.vstack(list(Y_test.values()))
regions = list(df_train.Unique_Region.unique())
 for region in regions:
selected_df = df_train[df_train.Unique_Region == region]
data_df = selected_df.iloc[71:84]
confirmed = list(data_df["ConfirmedCases"])
 fatality = list(data_df["Fatalities"])
 submit_from_pred(Date_submission, confirmed, region, df_test, df_sub, 0)
 submit_from_pred(Date_submission, fatality, region, df_test, df_sub, 1)
<save_to_csv>
y_train = Y_train[:, 0]
y_test = Y_test[:, 0]
ridge_cc = Pipeline(
 [ 
('ss', StandardScaler()), 
('ridge', RidgeCV())
 ] 
)
 ridge_cc.fit(X_train, y_train)
 print(ridge_cc.score(X_train, y_train))
print(ridge_cc.score(X_test, y_test))
df_sub.to_csv("submission.csv", index = False )<import_modules>
y_train = Y_train[:, 1]
y_test = Y_test[:, 1]
ridge_f = Pipeline(
 [ 
('ss', StandardScaler()), 
('ridge', RidgeCV())
 ] 
)
 ridge_f.fit(X_train, y_train)
 print(ridge_f.score(X_train, y_train))
print(ridge_f.score(X_test, y_test))
import os, traceback
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
from sklearn.linear_model import LinearRegression, Ridge
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import PolynomialFeatures, StandardScaler
from sklearn.svm import LinearSVR
from sklearn.utils import shuffle 
<load_from_csv>
test_data = pd.read_csv(
 '/kaggle/input/covid19-global-forecasting-week-4/test.csv'
).set_index('ForecastId')
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
path = os.path.join(dirname, filename)
 if 'train' in filename:
train = pd.read_csv(path)
 elif 'test' in filename:
test = pd.read_csv(path)
 scaler = StandardScaler() 
<data_type_conversions>
test_data = transform_area(test_data)
 test_data = transform_countries(test_data)
 test_data.head()
train.rename(columns={'Country_Region':'Country',
'Province_State': 'State'},
inplace=True)
 test.rename(columns={'Country_Region':'Country',
'Province_State': 'State'},
inplace=True)
 train['Date'] = pd.to_datetime(train['Date'], infer_datetime_format=True)
 test['Date'] = pd.to_datetime(test['Date'], infer_datetime_format=True)
 submission = pd.DataFrame(columns=['ForecastId','ConfirmedCases','Fatalities'], dtype='int32')
 df = pd.concat([train,test],sort=False)
 df['Country'] = df.Country.fillna("None")
 df['State'] = df.State.fillna("None")
<categorify>
test_data['ConfirmedCases'] = np.NaN
test_data['Fatalities'] = np.NaN
class BellModel() :
def __init__(self,streak_threshold=7,power=10):
self.streak_threshold = streak_threshold
self.power = power
def __str__(self):
return "BellModel(streak_threshold={},power={},
popt={})".format(
 self.streak_threshold,
self.power,
self.popt 
)
 def func(self,X,a,b,c):
result = a*np.exp(-(((X-b)**2)/(2*(c**2+0.1))))
return result.flatten()
def _find_streak(self, arr):
pos = np.clip(arr, 0, 1 ).astype(bool ).cumsum()
neg = np.clip(arr, -1, 0 ).astype(bool ).cumsum()
streaks = np.where(arr >= 0,
pos-np.maximum.accumulate(np.where(arr <= 0, pos, 0)) ,
-neg+np.maximum.accumulate(np.where(arr >= 0, neg, 0)))
 return streaks
def fit(self,x,y):
self.streak_threshold = -self.streak_threshold
x = scaler.inverse_transform(x)
 y = y.T.flatten() ; x = x.T.flatten()
c = np.argsort(x[:]); y = y[c]
x = np.sort(x)
 nonzero_index, *_ = np.where(np.sign(y ).cumsum() == 1)
 if not isinstance(nonzero_index,np.ndarray)and not nonzero_index:
self.popt = [0,0,0]; return
else:
try:
nonzero_index = nonzero_index.item(0)
 except IndexError:
self.popt = [0,0,0]; return
zeros, y_sp = y[:nonzero_index],y[nonzero_index:]
a = b = d = 0; c = 1
a_min = b_min = 0; c_min = 1
a_min = np.max(y_sp)
 a_max = a_min*self.power+1
d = len(zeros)
 y1 = np.diff(y_sp)
 y1_streaks = self._find_streak(y1)
 if self.streak_threshold in y1_streaks:
a = np.max(y_sp)
 else:
b_min = len(y_sp)+ d
y2 = np.diff(y1)
 y2_streaks = self._find_streak(y2)
 if self.streak_threshold in y2_streaks:
b = np.where(y2_streaks==self.streak_threshold)[0][0] * 2
c = b/3
else:
c_min = len(y_sp)*2/3
a = a_min if a < a_min else a
b = 300 if b < b_min else b
c = 100 if c < c_min else c
try:
self.popt, self.pconv = curve_fit(
 self.func, x, y,
maxfev=100000,
check_finite=False,
p0=[a,b,c],
bounds=([a_min,b_min,c_min],[a_max,300.,100.])
)
 except Exception as e:
traceback.print_exc()
self.popt = [0,0,0]
return self
def get_params(self, deep=False):
return {
'streak_threshold': self.streak_threshold,
'power': self.power,
}
def set_params(self, **params):
for param, value in params.items() :
setattr(self, param, value)
 return self
def get_estimated_params(self):
return [self.streak_threshold, self.power]
def set_curve_params(self, streak_threshold, power):
self.streak_threshold = streak_threshold
self.power = power
return self
def predict(self,x):
x = scaler.inverse_transform(x ).T.flatten()
return self.func(x,*self.popt)
<train_model>
train_dates = set(train_data.Date)
 test_dates = set(test_data.Date)
 for i in test_data.loc[test_data.Date <= last_date].index:
date = test_data.loc[i].Date
if date in set(train_data.Date):
slc = train_data.loc[(train_data.Date == date)&(train_data.Area == test_data.loc[i, "Area"])]
test_data.loc[i, "ConfirmedCases"] = slc["ConfirmedCases"].iloc[0]
test_data.loc[i, "Fatalities"] = slc["Fatalities"].iloc[0]
def model_fit(X_train, Y_train, X_test):

 X_train = scaler.fit_transform(X_train.reshape(-1,1))
X_test = scaler.transform(X_test.reshape(-1,1))
X_train_sh, Y_train_sh = shuffle(X_train,Y_train, random_state=0)
 remove_neg = lambda x: np.rint(x ).astype(int ).clip(min=0)
 metric = "neg_mean_squared_error"
param_grid = {
'streak_threshold': [5,6,7,14,21],
'power': [2,3,5,10,20,30,50]
}
bell_grid = GridSearchCV(BellModel() ,
param_grid,
cv=3,
scoring=metric,
verbose=0 
)
 bell_grid.fit(X_train_sh, Y_train_sh)
 bell_model = bell_grid.best_estimator_
param_grid = {
'polynomialfeatures__degree': np.arange(1,5),
'ridge__alpha':[1e2, 1e3,1e4],
'ridge__fit_intercept': [True, False],
'ridge__normalize': [True, False],
'ridge__solver': ['auto', 'svd', 'cholesky', 'lsqr', 'sparse_cg']
}
def PolynomialRidgeRegression(degree=2, **kwargs):
return make_pipeline(PolynomialFeatures(degree), Ridge(**kwargs))
ridge_grid = GridSearchCV(PolynomialRidgeRegression() ,
param_grid,
cv=3,
scoring=metric,
verbose=0)
 ridge_grid.fit(X_train_sh, Y_train_sh)
 ridge_model = ridge_grid.best_estimator_
param_grid = {
'polynomialfeatures__degree': np.arange(1,5),
'linearsvr__C' : np.logspace(0,1,5),
'linearsvr__epsilon' : np.logspace(-1,1,5),
'linearsvr__fit_intercept': [True, False],
}
def PolynomialSVRRegression(degree=2, **kwargs):
return make_pipeline(PolynomialFeatures(degree), LinearSVR(**kwargs,max_iter=10000))
svr_grid = GridSearchCV(PolynomialSVRRegression() ,
param_grid,
cv=3,
scoring=metric,
verbose=0)
 svr_grid.fit(X_train_sh, Y_train_sh)
 svr_model = svr_grid.best_estimator_
X_train2 = pd.DataFrame({'Bell': remove_neg(bell_model.predict(X_train)) ,
'Ridge': remove_neg(ridge_model.predict(X_train)) ,
'SVR': remove_neg(svr_model.predict(X_train)) ,
})
 X_test2 = pd.DataFrame({ 'Bell': remove_neg(bell_model.predict(X_test)) ,
'Ridge': remove_neg(ridge_model.predict(X_test)) ,
'SVR': remove_neg(svr_model.predict(X_test)) ,
})
 reg = LinearRegression()
reg.fit(X_train2, Y_train)
 Y_test = reg.predict(X_test2)
 Y_test = remove_neg(Y_test)
 return Y_test 
<prepare_x_and_y>
for area in sorted(list(set(test_data.Area))):
print(area)
 area_data = test_data.loc[test_data.Area == area].set_index('Date' ).sort_index()
for i in area_data.index:
if not np.isnan(area_data.loc[i, "ConfirmedCases"])and not np.isnan(area_data.loc[i, "Fatalities"]):
continue
x = np.zeros(2 * N_FEATURES + 2)
 for j in range(1, 1 + N_FEATURES):
x[j * 2 - 2] = area_data.shift(j ).loc[i].ConfirmedCases
x[j * 2 - 1] = area_data.shift(j ).loc[i].Fatalities
x[2 * N_FEATURES] = area_data.loc[i].Density
x[2 * N_FEATURES + 1] = area_data.loc[i].InfantMortality
x = x.reshape(1, -1)
 test_data.loc[(test_data.Area == area)&(test_data.Date == i), "ConfirmedCases"] = ridge_cc.predict(x)[0] + area_data.shift(1 ).loc[i, "ConfirmedCases"]
test_data.loc[(test_data.Area == area)&(test_data.Date == i), "Fatalities"] = ridge_f.predict(x)[0] + area_data.shift(1 ).loc[i, "Fatalities"]
area_data.loc[i, "ConfirmedCases"] = ridge_cc.predict(x)[0] + area_data.shift(1 ).loc[i, "ConfirmedCases"]
area_data.loc[i, "Fatalities"] = ridge_f.predict(x)[0] + area_data.shift(1 ).loc[i, "Fatalities"]
for name,state in df.groupby(['Country','State']):
mask = state.duplicated(subset=['Date'],keep='first')
 df_tmp = state.loc[~mask]
df_tmp_dropped = state.loc[mask]
drop_num = len(state)- len(df_tmp)
 try:
df_tmp.insert(0,'Index', range(1,len(df_tmp)+1))
except:
pass
tmp_train = df_tmp.dropna(subset=['ConfirmedCases'])
 X_tr = tmp_train['Index'].values
tmp_test = df_tmp[ df_tmp['ConfirmedCases'].isna() ]
X_te = tmp_test.Index.values
for cat in('ConfirmedCases', 'Fatalities'):
Y_tr = tmp_train[cat].values
Y_te = model_fit(X_tr, Y_tr, X_te)
 tmp_test.loc[:,cat] = Y_te
tmp_train = pd.merge(tmp_train.tail(drop_num)[['Date','ConfirmedCases','Fatalities']],
df_tmp_dropped[['Date','ForecastId']],
on='Date')
 col = ['ForecastId','ConfirmedCases','Fatalities']
tmp_train = tmp_train[col]
tmp_test = tmp_test[col]
tmp = pd.concat([tmp_train,tmp_test])
 submission = pd.concat([submission, tmp])
 os.chdir("/kaggle/working/")
 submission = submission.fillna(0)
 submission = submission.astype(int)
 submission.to_csv('submission.csv', index=False)
 plot_data(train,test,submission)
<import_modules>
submission = pd.read_csv(
 "/kaggle/input/covid19-global-forecasting-week-4/submission.csv"
).set_index('ForecastId')
t0 = time.time()<load_from_csv>
submission['ConfirmedCases'] = test_data.loc[submission.index, "ConfirmedCases"]
data_path = '.. /input/'
train_df = pd.read_csv(data_path + 'train.csv')
 test_df = pd.read_csv(data_path + 'test.csv')
 print('Train : ', train_df.shape)
 print('Test : ', test_df.shape )<init_hyperparams>
submission['Fatalities'] = test_data.loc[submission.index, "Fatalities"]
seed = 2077
max_len = 50
lower = False
trunc = 'pre'
max_features = 120000
n_vocab = max_features
clean_num = 0
n_models = 6
epochs = 8
batch_size = 512
drop_last = True
hidden_dim = 128
fix_embedding = True
unk_uni = True
n_embed = 2
embed_dim = n_embed * 300
proj_dim = hidden_dim
bidirectional = True
n_layers = 1
rnn_dim = hidden_dim
dense_dim = 2 * rnn_dim if bidirectional else rnn_dim
mu = 0.9
updates_per_epoch = 10
threshold = 0.37
test_batch_size = 8 * batch_size<set_options>
submission.loc[submission.ConfirmedCases < 0, "ConfirmedCases"] = 0
submission.loc[submission.Fatalities < 0, "Fatalities"] = 0
def seed_torch(seed=1):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_torch(seed)
 device = torch.device('cuda')if torch.cuda.is_available() else 'cpu'
def get_param_size(model, trainable=True):
if trainable:
psize = np.sum([np.prod(p.size())for p in model.parameters() if p.requires_grad])
 else:
psize = np.sum([np.prod(p.size())for p in model.parameters() ])
 return psize
class EMA() :
def __init__(self, model, mu, level='batch', n=1):

 self.mu = mu
self.level = level
self.n = n
self.cnt = self.n
self.shadow = {}
for name, param in model.named_parameters() :
if param.requires_grad:
self.shadow[name] = param.data
def _update(self, model):
for name, param in model.named_parameters() :
if param.requires_grad:
new_average =(1 - self.mu)* param.data + self.mu * self.shadow[name]
self.shadow[name] = new_average.clone()
def set_weights(self, ema_model):
for name, param in ema_model.named_parameters() :
if param.requires_grad:
param.data = self.shadow[name]
def on_batch_end(self, model):
if self.level is 'batch':
self.cnt -= 1
if self.cnt == 0:
self._update(model)
 self.cnt = self.n
def on_epoch_end(self, model):
if self.level is 'epoch':
self._update(model)
 class GlobalMaxPooling1D(nn.Module):
def __init__(self):
super(GlobalMaxPooling1D, self ).__init__()
def forward(self, inputs):
z, _ = torch.max(inputs, 1)
 return z
def __repr__(self):
return self.__class__.__name__ + '() '
class GRUModel(nn.Module):
def __init__(self, n_vocab, embed_dim, proj_dim, rnn_dim, n_layers, bidirectional, dense_dim,
padding_idx=0, pretrained_embedding=None, fix_embedding=True,
n_out=1):
super(GRUModel, self ).__init__()
self.n_vocab = n_vocab
self.embed_dim = embed_dim
self.n_layers = n_layers
self.dense_dim = dense_dim
self.n_out = n_out
self.bidirectional = bidirectional
self.fix_embedding = fix_embedding
self.padding_idx = padding_idx
if pretrained_embedding is not None:
self.embed = nn.Embedding.from_pretrained(pretrained_embedding, freeze=fix_embedding)
 self.embed.padding_idx = self.padding_idx
else:
self.embed = nn.Embedding(self.n_vocab, self.embed_dim, padding_idx=self.padding_idx)
 self.proj = nn.Linear(embed_dim, proj_dim)
 self.proj_act = nn.ReLU()
self.gru = nn.GRU(proj_dim, rnn_dim, self.n_layers,
batch_first=True, bidirectional=bidirectional)
 self.pooling = GlobalMaxPooling1D()
in_dim = 2 * rnn_dim if self.bidirectional else rnn_dim
self.dense = nn.Linear(in_dim, dense_dim)
 self.dense_act = nn.ReLU()
self.out_linear = nn.Linear(dense_dim, n_out)
 self.init_weights()
def init_weights(self):
for name, param in self.named_parameters() :
if name.find('embed')> -1:
continue
elif name.find('weight')> -1 and len(param.size())> 1:
nn.init.xavier_uniform_(param)
 def forward(self, inputs):
x = self.embed(inputs)
 x = self.proj_act(self.proj(x))
x, hidden = self.gru(x)
 x = self.pooling(x)
 x = self.dense_act(self.dense(x))
x = self.out_linear(x)
 return x
def fit(self, dataloader, epochs, optimizer, callbacks=None):
for i in range(epochs):
run_epoch(model, dataloader, optimizer, callbacks=callbacks)
 def predict(self, dataloader):
preds = []
with torch.no_grad() :
for batch in dataloader:
batch = tuple(t.to(device)for t in batch)
 X_batch = batch[0]
preds.append(self.forward(X_batch ).data.cpu())
 return torch.cat(preds)
 def predict_proba(self, dataloader):
return torch.sigmoid(self.predict(dataloader)).data.numpy()<categorify>
submission['ConfirmedCases'] = submission.ConfirmedCases.astype(int)
 submission['Fatalities'] = submission.Fatalities.astype(int )
class TextClassificationDataset(Dataset):

 def __init__(self, token_ids_s, labels=None, max_len=1000, training=True, sort=True):
self.training = training
if labels is None:
self.labels = torch.ones(len(token_ids_s), dtype=torch.long)
 else:
self.labels = torch.LongTensor(labels)
 seq_lens = []
self.inputs = []
for e, token_ids in enumerate(token_ids_s):
seq_lens.append(len(token_ids))
input_ids = torch.LongTensor(token_ids[:max_len])
 self.inputs.append(input_ids)
 if self.training and sort:
self.indices = np.argsort(seq_lens)
 self.inputs = [self.inputs[i] for i in self.indices]
self.labels = self.labels[self.indices]
def __len__(self):
return len(self.inputs)
 def __getitem__(self, idx):
return self.inputs[idx], self.labels[idx]
def set_labels(self, labels):
self.labels = torch.tensor(labels, dtype=torch.long)
 class BatchIterator(object):
def __init__(self, dataset, collate_fn, batch_size,
shuffle=True, drop_last=True):
self.dataset = dataset
self.collate_fn = collate_fn
self.batch_size = batch_size
self.size = len(dataset)
 self.shuffle = shuffle
if drop_last:
self.num_batches = self.size // batch_size
else:
self.num_batches =(self.size + self.batch_size - 1)// self.batch_size
def __iter__(self):
if self.shuffle:
indices = np.random.choice(self.num_batches, self.num_batches, replace=False)
 else:
indices = range(self.num_batches)
 for idx in indices:
left = self.batch_size * idx
yield(self.collate_fn(self.dataset[left: left + self.batch_size]))
def __len__(self):
return self.num_batches
def collate_fn(batch):
xy_batch = [pad_sequence(batch[0], batch_first=True), batch[1]]
return xy_batch
def run_epoch(model, dataloader, optimizer, callbacks=None,
criterion=nn.BCEWithLogitsLoss() , verbose_step=10000):
t1 = time.time()
tr_loss = 0
for step, batch in enumerate(dataloader):
batch = tuple(t.to(device)for t in batch)
 x_batch, y_batch = batch
model.zero_grad()
outputs = model(x_batch)
 loss = criterion(outputs[:, 0], y_batch.float())
 loss.backward()
optimizer.step()
tr_loss += loss.item()
if callbacks is not None:
for func in callbacks:
func.on_batch_end(model)
 if(step + 1)% verbose_step == 0:
loss_now = tr_loss /(step + 1)
 print(f'step:{step+1} loss:{loss_now:.7f} time:{time.time() - t1:.1f}s')
 if callbacks is not None:
for func in callbacks:
func.on_epoch_end(model)
 return tr_loss /(step + 1 )<compute_train_metric>
submission.to_csv('submission.csv')
def load_glove(word_index, max_features, unk_uni):
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 EMBEDDING_FILE = data_path + 'embeddings/glove.840B.300d/glove.840B.300d.txt'
embeddings_index = dict(get_coefs(*o.split(' ')) for o in open(EMBEDDING_FILE))
all_embs = np.stack(list(embeddings_index.values()))
emb_mean, emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
unknown_words = []
nb_words = min(max_features, len(word_index))
if unk_uni:
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
else:
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= nb_words:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is None:
embedding_vector = embeddings_index.get(word.lower())
 if embedding_vector is None:
unknown_words.append(( word, i))
else:
embedding_matrix[i] = embedding_vector
else:
embedding_matrix[i] = embedding_vector
print('
Total unknowns glove', len(unknown_words))
print(unknown_words[:10])
 del embeddings_index
gc.collect()
return embedding_matrix, unknown_words
def load_wiki(word_index, max_features, unk_uni):
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 EMBEDDING_FILE = data_path + 'embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
embeddings_index = dict(get_coefs(*o.split(' ')) for o in open(EMBEDDING_FILE)if len(o)> 100)
 all_embs = np.stack(list(embeddings_index.values()))
emb_mean, emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
unknown_words = []
nb_words = min(max_features, len(word_index))
if unk_uni:
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
else:
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= nb_words:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is None:
embedding_vector = embeddings_index.get(word.lower())
 if embedding_vector is None:
unknown_words.append(( word, i))
else:
embedding_matrix[i] = embedding_vector
else:
embedding_matrix[i] = embedding_vector
print('
Total unknowns wiki', len(unknown_words))
print(unknown_words[:10])
 del embeddings_index
gc.collect()
return embedding_matrix, unknown_words
def load_parag(word_index, max_features, unk_uni):
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 EMBEDDING_FILE = data_path + 'embeddings/paragram_300_sl999/paragram_300_sl999.txt'
embeddings_index = dict(get_coefs(*o.split(' '))
for o in open(EMBEDDING_FILE, encoding='utf8', errors='ignore')
 if len(o)> 100)
 all_embs = np.stack(list(embeddings_index.values()))
emb_mean, emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
unknown_words = []
nb_words = min(max_features, len(word_index))
if unk_uni:
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
else:
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= nb_words:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is None:
embedding_vector = embeddings_index.get(word.lower())
 if embedding_vector is None:
unknown_words.append(( word, i))
else:
embedding_matrix[i] = embedding_vector
else:
embedding_matrix[i] = embedding_vector
print('
Total unknowns parag', len(unknown_words))
print(unknown_words[:10])
 del embeddings_index
gc.collect()
return embedding_matrix, unknown_words
def load_ggle(word_index, max_features, unk_uni):
EMBEDDING_FILE = data_path + 'embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)
 embed_size = embeddings_index.get_vector('known' ).size
unknown_words = []
nb_words = min(max_features, len(word_index))
if unk_uni:
embedding_matrix =(np.random.rand(nb_words, embed_size)- 0.5)/ 5.0
else:
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= nb_words:
continue
if word in embeddings_index:
embedding_vector = embeddings_index.get_vector(word)
 embedding_matrix[i] = embedding_vector
else:
word_lower = word.lower()
if word_lower in embeddings_index:
embedding_matrix[i] = embeddings_index.get_vector(word_lower)
 else:
unknown_words.append(( word, i))
print('
Total unknowns ggle', len(unknown_words))
print(unknown_words[:10])
 del embeddings_index
gc.collect()
return embedding_matrix, unknown_words
def load_all_embeddings(tokenizer, max_features, clean_num=False, unk_uni=True):
word_index = tokenizer.word_index
if clean_num == 2:
ggle_word_index = {}
for word, i in word_index.items() :
ggle_word_index[clean_numbers(word)] = i
else:
ggle_word_index = word_index
embedding_matrix_1, u1 = load_glove(word_index, max_features, unk_uni)
 embedding_matrix_2, u2 = load_wiki(word_index, max_features, unk_uni)
 embedding_matrix_3, u3 = load_parag(word_index, max_features, unk_uni)
 embedding_matrix_4, u4 = load_ggle(ggle_word_index, max_features, unk_uni)
 embedding_matrix = np.concatenate(( embedding_matrix_1,
embedding_matrix_2,
embedding_matrix_3,
embedding_matrix_4), axis=1)
 del embedding_matrix_1, embedding_matrix_2, embedding_matrix_3, embedding_matrix_4
gc.collect()
return embedding_matrix
def setup_emb(tr_X, max_features=50000, clean_num=2, unk_uni=True):
tokenizer = Tokenizer(num_words=max_features, lower=False, filters='')
 tokenizer.fit_on_texts(tr_X)
 print('len(vocab)', len(tokenizer.word_index))
embedding_matrix = load_all_embeddings(tokenizer, max_features=max_features,
clean_num=clean_num, unk_uni=unk_uni)
 return tokenizer, embedding_matrix<categorify>
test_data.loc[test_data.Area == 'russia']