Kaggle competition: Language Identification
Nlim = 5
delta_max = 0.1
def calc_trends(X_train,y_true,X_test,deg):
pct_hist = 100*sum(1 for x in y_true if x > 0)/N_train
y=np.array(y_true ).flatten()
y_last = y[len(y)-1]
y_check = y[len(y)-1-Nlim]
if(y_check==0):
den = 1
else :
den = y_check
diff = np.abs(y_last-y_check)/den
lim = np.mean(y_true[len(y_true)-Nlim:len(y_true)])
 if(diff<=delta_max):
y_valid = np.full([len(y_true)],lim)
 y_pred = np.full([len(X_test)],lim)
 msg1 = 'cost'
msg2 = 'cost'
if(diff>delta_max):
y_valid,y_pred = polynomial_trend(deg,X_train,y_true,X_test)
 msg1 = 'poly'
msg2 = 'poly'
return y_valid, y_pred, msg1, msg2<feature_engineering>
class Dictionary(object):
def __init__(self):
self.token2idx = {}
self.idx2token = []
def add_token(self, token):
if token not in self.token2idx:
self.idx2token.append(token)
 self.token2idx[token] = len(self.idx2token)- 1
return self.token2idx[token]
def __len__(self):
return len(self.idx2token )
df_train = df_union[df_union['DsType']=='train'].drop('DsType',axis=1)
 df_test = df_union[df_union['DsType']=='test'].drop('DsType',axis=1)
 df_train['ConfirmedCasesValid'] = 0
df_train['FatalitiesValid'] = 0
df_train['ConfirmedCasesTrend'] = 0
df_train['FatalitiesTrend'] = 0
df_train['ConfirmedCasesResid'] = 0
df_train['FatalitiesResid'] = 0
df_test['ConfirmedCases'] = 0
df_test['Fatalities'] = 0
df_test['ConfirmedCasesTrend'] = 0
df_test['FatalitiesTrend'] = 0
df_test['ConfirmedCasesResid'] = 0
df_test['FatalitiesResid'] = 0<data_type_conversions>
char_vocab = Dictionary()
pad_token = '<pad>'
unk_token = '<unk>'
pad_index = char_vocab.add_token(pad_token)
 unk_index = char_vocab.add_token(unk_token)
 chars = set(''.join(x_train_full))
for char in sorted(chars):
char_vocab.add_token(char)
 print("Vocabulary:", len(char_vocab), "UTF characters")
 lang_vocab = Dictionary()
languages = set(y_train_full)
 for lang in sorted(languages):
lang_vocab.add_token(lang)
 print("Labels:", len(lang_vocab), "languages" )
y1_train = df_train['ConfirmedCases'].astype(float)
 y2_train = df_train['Fatalities'].astype(float )<filter>
print('a ->', char_vocab.token2idx['a'])
 print('cat ->', lang_vocab.token2idx['cat'])
 print(y_train_full[0], x_train_full[0][:10])
 x_train_idx = [np.array([char_vocab.token2idx[c] for c in line])for line in x_train_full]
y_train_idx = np.array([lang_vocab.token2idx[lang] for lang in y_train_full])
 print(y_train_idx[0], x_train_idx[0][:10] )
def isPos(x):
if(x>0):
return 1
else :
return 0<feature_engineering>
x_train, x_val, y_train, y_val = train_test_split(x_train_idx, y_train_idx, test_size=0.15, random_state=seed)
 train_data = [(x, y)for x, y in zip(x_train, y_train)]
val_data = [(x, y)for x, y in zip(x_val, y_val)]
print(len(train_data), "training samples")
 print(len(val_data), "validation samples" )
df_train['ConfirmedCasesPos']=df_train['ConfirmedCases'].apply(isPos)
 df_train['FatalitiesPos']=df_train['Fatalities'].apply(isPos)
 df_test['ConfirmedCasesPos']=df_test['ConfirmedCases'].apply(isPos)
 df_test['FatalitiesPos']=df_test['Fatalities'].apply(isPos )<groupby>
def batch_generator(data, batch_size, token_size):

 minibatch, sequences_so_far, tokens_so_far = [], 0, 0
for ex in data:
minibatch.append(ex)
 seq_len = len(ex[0])
 if seq_len > token_size:
ex =(ex[0][:token_size], ex[1])
 seq_len = token_size
sequences_so_far += 1
tokens_so_far += seq_len
if sequences_so_far == batch_size or tokens_so_far == token_size:
yield minibatch
minibatch, sequences_so_far, tokens_so_far = [], 0, 0
elif sequences_so_far > batch_size or tokens_so_far > token_size:
yield minibatch[:-1]
minibatch, sequences_so_far, tokens_so_far = minibatch[-1:], 1, len(minibatch[-1][0])
 if minibatch:
yield minibatch
sel_cols = ['Country_Region','Province_State','ConfirmedCasesPos','FatalitiesPos']
df_tr_cnt = df_train[sel_cols].groupby(by=['Country_Region','Province_State'] ).sum()
df_te_cnt = df_test[sel_cols].groupby(by=['Country_Region','Province_State'] ).sum()<filter>
def pool_generator(data, batch_size, token_size, shuffle=False):

 for p in batch_generator(data, batch_size * 100, token_size * 100):
p_batch = batch_generator(sorted(p, key=lambda t: len(t[0]), reverse=True), batch_size, token_size)
 p_list = list(p_batch)
 if shuffle:
for b in random.sample(p_list, len(p_list)) :
yield b
else:
for b in p_list:
yield b
sel_cols = ['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']
df_tr_last = df_train[df_train['Date']==max(df_train['Date'])][sel_cols]
df_te_last = df_test[df_test['Date']==max(df_test['Date'])][sel_cols]<merge>
class CharRNNClassifier(torch.nn.Module):
def __init__(self, input_size, embedding_size, hidden_size, output_size, model="lstm", num_layers=1, bidirectional=False, pad_idx=0):
super().__init__()
self.model = model.lower()
self.hidden_size = hidden_size
self.embed = torch.nn.Embedding(input_size, embedding_size, padding_idx=pad_idx)
 if self.model == "gru":
self.rnn = torch.nn.GRU(embedding_size, hidden_size, num_layers, bidirectional=bidirectional)
 elif self.model == "lstm":
self.rnn = torch.nn.LSTM(embedding_size, hidden_size, num_layers, bidirectional=bidirectional)
 self.linear1 = torch.nn.Linear(hidden_size, output_size)
 self.linear2 = torch.nn.Linear(output_size, input_size)
 def forward(self, input, input_lengths):
encoded = self.embed(input)
 packed = torch.nn.utils.rnn.pack_padded_sequence(encoded, input_lengths)
 output, _ = self.rnn(packed)
 padded, _ = torch.nn.utils.rnn.pad_packed_sequence(output, padding_value=float('-inf'))
padded = padded.permute(1,2,0)
 output = F.adaptive_max_pool1d(padded, 1 ).view(-1, self.hidden_size)
 output = self.linear1(output)
 output = self.linear2(output)
 return output
df_tr_last = df_tr_last.merge(df_tr_cnt,on=['Country_Region','Province_State'])
 df_te_last = df_te_last.merge(df_te_cnt,on=['Country_Region','Province_State'] )<merge>
if not torch.cuda.is_available() :
print("WARNING: CUDA is not available.Select 'GPU On' on kernel settings")
 device = torch.device("cuda")
 torch.cuda.manual_seed(seed )
df_last = df_tr_last.merge(df_te_last,on=['Country_Region','Province_State'])
 df_last['DeltaConfirmedCases']=100*(df_last['ConfirmedCases_y']-df_last['ConfirmedCases_x'])/df_last['ConfirmedCases_x']
df_last['DeltaFatalities']=100*(df_last['Fatalities_y']-df_last['Fatalities_x'])/df_last['Fatalities_x']
df_last['DeltaConfirmedCases']=df_last['DeltaConfirmedCases'].apply(np.abs)
 df_last['DeltaFatalities']=df_last['DeltaFatalities'].apply(np.abs)
 df_last.fillna(0,inplace=True)
 df_last = df_last.replace([np.inf, -np.inf], 0 )<sort_values>
criterion = torch.nn.CrossEntropyLoss(reduction='sum' )
n_head = 5
d_lim1 = 120
d_lim2 = 200
sel_var ='ConfirmedCases'
sel_cols = ['Country_Region','Province_State',sel_var+'_x','Delta'+sel_var,sel_var+'Pos_x']
condition =(df_last['Delta'+sel_var]>d_lim1)&(df_last['Delta'+sel_var]<d_lim2)
 df_last[condition][sel_cols].sort_values(by=['Delta'+sel_var], ascending=True ).head(n_head )<sort_values>
def train(model, optimizer, data, batch_size, token_size, log=False):
model.train()
total_loss = 0
ncorrect = 0
nsentences = 0
ntokens = 0
niterations = 0
for batch in pool_generator(data, batch_size, token_size, shuffle=True):
X = [torch.from_numpy(d[0])for d in batch]
X_lengths = [x.numel() for x in X]
ntokens += sum(X_lengths)
 X_lengths = torch.tensor(X_lengths, dtype=torch.long, device=device)
 y = torch.tensor([d[1] for d in batch], dtype=torch.long, device=device)
 X = torch.nn.utils.rnn.pad_sequence(X ).to(device)
 model.zero_grad()
output = model(X, X_lengths)
 loss = criterion(output, y)
 loss.backward()
optimizer.step()
total_loss += loss.item()
ncorrect +=(torch.max(output, 1)[1] == y ).sum().item()
nsentences += y.numel()
niterations += 1
total_loss = total_loss / nsentences
accuracy = 100 * ncorrect / nsentences
if log:
print(f'Train: wpb={ntokens//niterations}, bsz={nsentences//niterations}, num_updates={niterations}')
 return accuracy
n_head = 5
d_lim1 = 0
d_lim2 = 10
sel_var = 'Fatalities'
sel_cols = ['Country_Region','Province_State',sel_var+'_x','Delta'+sel_var,sel_var+'Pos_x']
condition =(df_last['Delta'+sel_var]>d_lim1)&(df_last['Delta'+sel_var]<d_lim2)
 df_last[condition][sel_cols].sort_values(by=['Delta'+sel_var], ascending=True ).head(n_head )<prepare_x_and_y>
def validate(model, data, batch_size, token_size):
model.eval()
ncorrect = 0
nsentences = 0
with torch.no_grad() :
for batch in pool_generator(data, batch_size, token_size):
X = [torch.from_numpy(d[0])for d in batch]
X_lengths = torch.tensor([x.numel() for x in X], dtype=torch.long, device=device)
 y = torch.tensor([d[1] for d in batch], dtype=torch.long, device=device)
 X = torch.nn.utils.rnn.pad_sequence(X ).to(device)
 answer = model(X, X_lengths)
 ncorrect +=(torch.max(answer, 1)[1] == y ).sum().item()
nsentences += y.numel()
dev_acc = 100 * ncorrect / nsentences
return dev_acc
def plotExample() :
train_cond =(( df_train['Country_Region']==my_country)&(df_train['Province_State']==my_province))
test_cond =(( df_test['Country_Region']==my_country)&(df_test['Province_State']==my_province))
x_train_plt = df_train[train_cond]['Date']
y_train_plt = df_train[train_cond][my_variable]
y_valid_plt = df_train[train_cond][my_variable+'Valid']
y_trend_plt = df_train[train_cond][my_variable+'Trend']
y_resid_plt = df_train[train_cond][my_variable+'Resid']
x_test_plt = df_test[test_cond]['Date']
y_test_plt = df_test[test_cond][my_variable]
plt.rcParams["figure.figsize"] =(12,6)
 fig, ax = plt.subplots()
ax.plot(x_train_plt,y_train_plt,'o',color='orange', label='y_true')
 ax.plot(x_train_plt,y_valid_plt,'x',color='gray' , label ='y_valid')
 ax.plot(x_train_plt,y_trend_plt,'.',color='lightblue', label='y_trend')
 ax.plot(x_train_plt,y_resid_plt,'-',color='green', label='y_resid')
 ax.plot(x_test_plt,y_test_plt,'*',color='red', label='y_test')
 ax.set_xticks([])
 ax.legend()<filter>
hidden_size = 256
embedding_size = 64
bidirectional = False
ntokens = len(char_vocab)
 nlabels = len(lang_vocab )
sel_cols = ['Country_Region','Province_State','ConfirmedCases_x','ConfirmedCases_y','DeltaConfirmedCases','ConfirmedCasesPos_x']
df_last[sel_cols][(( df_last['Country_Region']==my_country)&(df_last['Province_State']==my_province)) ]<filter>
def testParameters(model, optimizer, epochs, batch_size, token_size):
train_accuracy = []
valid_accuracy = []
for epoch in range(1, epochs + 1):
acc = train(model, optimizer, train_data, batch_size, token_size, log=epoch==1)
 train_accuracy.append(acc)
 acc = validate(model, val_data, batch_size, token_size)
 valid_accuracy.append(acc)
 return train_accuracy, valid_accuracy
sel_cols = ['Country_Region','Province_State','ConfirmedCases_x','ConfirmedCases_y','DeltaConfirmedCases','ConfirmedCasesPos_x']
df_last[sel_cols][(( df_last['Country_Region']==my_country)&(df_last['Province_State']==my_province)) ]<filter>
Hyperparameter optimisation:
Trains and vlidates the model, then returns the accuracy to persist over iterations .
sel_cols = ['Country_Region','Province_State','ConfirmedCases_x','ConfirmedCases_y','DeltaConfirmedCases','ConfirmedCasesPos_x']
df_last[sel_cols][(( df_last['Country_Region']==my_country)&(df_last['Province_State']==my_province)) ]<filter>
model = CharRNNClassifier(ntokens, embedding_size, hidden_size, nlabels, bidirectional=bidirectional, pad_idx=pad_index ).to(device)
 optimizer = torch.optim.Adam(model.parameters() )
sel_cols = ['Country_Region','Province_State','ConfirmedCases_x','ConfirmedCases_y','DeltaConfirmedCases','ConfirmedCasesPos_x']
df_last[sel_cols][(( df_last['Country_Region']==my_country)&(df_last['Province_State']==my_province)) ]<prepare_x_and_y>
batch_sizes = [64, 128, 256, 512, 1024]
token_sizes = np.arange(50000, 350000, 50000)
 hidden_sizes = np.arange(128, 768, 64)
 embedding_sizes = np.arange(64, 256*4, 64)
 embedding_size = 128
hidden_size = 512
hidden_models = []
hidden_optimizers = []
figure = 0
batch_size = 256
token_size = 150000
epochs = 20
max_acc = 0
std_epochs = 10
bidirectional = False
results = testParameters(model,optimizer, epochs, batch_size, token_size)
 plt.figure(figure)
 plt.plot(range(1, len(results[0])+1), results[0])
 plt.plot(range(1, len(results[1])+1), results[1])
 plt.title('Acuracy')
 plt.xlabel('epoch')
 plt.ylabel('Accuracy');
y1_valid = cut_neg(df_train['ConfirmedCasesValid'].copy())
 y2_valid = cut_neg(df_train['FatalitiesValid'].copy())
 y1_pred = cut_neg(df_test['ConfirmedCases'].copy())
 y2_pred = cut_neg(df_test['Fatalities'].copy() )<compute_test_metric>
mse =(mean_squared_log_error(y1_train, y1_valid)+mean_squared_log_error(y2_train, y2_valid)) /2
mse<prepare_output>
submission = pd.DataFrame({'ForecastId': df_test['ForecastId'],'ConfirmedCases': y1_pred,'Fatalities': y2_pred})
 submission.head()<save_to_csv>
model = CharRNNClassifier(ntokens, embedding_size, hidden_size, nlabels, bidirectional=bidirectional, pad_idx=pad_index ).to(device)
 optimizer = torch.optim.Adam(model.parameters() )
submission.to_csv('submission.csv',index = False )<compute_test_metric>
model = CharRNNClassifier(ntokens, embedding_size, hidden_size, nlabels, bidirectional=bidirectional ).to(device)
 optimizer = torch.optim.Adam(model.parameters() )
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
def fix_target(frame, key, target, new_target_name="target"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
for i in range(1, len(group_keys)- 1):
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if current_value<previous_value:
current_value=previous_value
target[i] =current_value
target[i] =max(0,target[i])
 frame[new_target_name] = np.array(target)
 def rate(frame, key, target, new_target_name="rate"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
rate=[1.0 for k in range(len(target)) ]
for i in range(1, len(group_keys)) :
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if previous_value!=0.0:
rate[i]=current_value/previous_value
rate[i] =max(1,rate[i])
 frame[new_target_name] = np.array(rate)
 def difference(frame, key, target, new_target_name="diff"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
rate=[0 for k in range(len(target)) ]
for i in range(1, len(group_keys)) :
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
rate[i]=np.log1p(max(0,current_value-previous_value))
rate[i] =max(0,rate[i])
 frame[new_target_name] = np.array(rate)
 def get_data_by_key(dataframe, key, key_value, fields=None):
mini_frame=dataframe[dataframe[key]==key_value]
if not fields is None:
mini_frame=mini_frame[fields].values
return mini_frame
directory="/kaggle/input/covid19-global-forecasting-week-4/"
model_directory="model"
us_county="/kaggle/input/covid19-us-county-jhu-data-demographics/covid_us_county"
geo_dir=None
extra_stable_columns=None
group_by_columns=None
if not group_by_columns is None and "continent" in group_by_columns:
assert not geo_dir is None
train=pd.read_csv(directory + "train.csv", parse_dates=["Date"] , engine="python")
 test=pd.read_csv(directory + "test.csv", parse_dates=["Date"], engine="python")
 train["key"]=train[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 test["key"]=test[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 if not geo_dir is None:
region_metadata=pd.read_csv(geo_dir+"region_metadata.csv")
 km=KMeans(n_clusters=30, init='k-means++', n_init=10, max_iter=300, tol=0.0001,
precompute_distances='auto', verbose=0, random_state=1234, copy_x=True, n_jobs=None, algorithm='auto')
 region_metadata["clusters"]=km.fit_predict(region_metadata[["lat","lon"]].values)
 train=pd.merge(train,region_metadata, how="left", left_on=["Province_State","Country_Region"], right_on=["Province_State","Country_Region"])
 test=pd.merge(test,region_metadata, how="left", left_on=["Province_State","Country_Region"], right_on=["Province_State","Country_Region"])
 group_names=None
if not group_by_columns is None and len(group_by_columns)>0:
group_names=[]
for group in group_by_columns:
groupss=train[["Date", group,target1,target2]]
grp=groupss.groupby(["Date", group], as_index=False ).sum()
grp.columns=["Date", group,group +"_" + target1,group +"_" + target2 ]
train=pd.merge(train,grp, how="left", left_on=["Date", group,group], right_on=["Date", group,group])
 for gr in [group +"_" + target1,group +"_" + target2]:
rate(train, key, gr, new_target_name="rate_" +gr)
 group_names+=["rate_" +gr]
train.to_csv(directory + "train_plus_groups.csv", index=False)
 max_train_date=train["Date"].max()
max_test_date=test["Date"].max()
horizon=(max_test_date-max_train_date ).days
print("horizon", int(horizon))
target1="ConfirmedCases"
target2="Fatalities"
key="key"
train_supplamanteary2=None
new_data_file2=[us_county]
if not new_data_file2 is None:
trains=pd.read_csv(directory + "train.csv", parse_dates=["Date"] , engine="python")
 tests=pd.read_csv(directory + "test.csv", parse_dates=["Date"], engine="python")
 trains["key"]=train[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 tests["key"]=test[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 train_supplamanteary2=None
for new_data_files in new_data_file2:
sumplementary_data=pd.read_csv(new_data_files +".csv" , parse_dates=["date"] , engine="python")
 sumplementary_data.columns=["fips","Province_State","Country_Region","lat","long","Date",
"ConfirmedCases","state_code","Fatalities"]
sumplementary_data["key"]=sumplementary_data[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 sumplementary_data=sumplementary_data[["Date","key","ConfirmedCases","Fatalities","Province_State","Country_Region"]]
sumplementary_data["Id"]=[k for k in range(999999999,sumplementary_data.shape[0]+999999999)]
sumplementary_data=sumplementary_data[sumplementary_data.Date>= '2020-02-20']
if train_supplamanteary2 is None:
train_supplamanteary2=pd.concat([trains.reset_index(drop=True), sumplementary_data.reset_index(drop=True)], axis=0 ).reset_index(drop=True)
 else :
train_supplamanteary2=pd.concat([train_supplamanteary2.reset_index(drop=True), sumplementary_data.reset_index(drop=True)], axis=0 ).reset_index(drop=True)
 print(train_supplamanteary2.head(5))
train_supplamanteary2 = train_supplamanteary2.sort_values(["Country_Region","key","Id"], ascending =(True, True, True))
fix_target(train_supplamanteary2, key, target1, new_target_name=target1)
 fix_target(train_supplamanteary2, key, target2, new_target_name=target2)
 rate(train_supplamanteary2, key, target1, new_target_name="rate_" +target1)
 rate(train_supplamanteary2, key, target2, new_target_name="rate_" +target2)
 difference(train_supplamanteary2, key, target1, new_target_name="diff_" +target1)
 difference(train_supplamanteary2, key, target2, new_target_name="diff_" +target2)
 unique_keys_sup2=train_supplamanteary2[key].unique()
print(len(unique_keys_sup2))
if not geo_dir is None:
region_metadata=pd.read_csv(geo_dir+"region_metadata.csv")
 km=KMeans(n_clusters=30, init='k-means++', n_init=10, max_iter=300, tol=0.0001,
precompute_distances='auto', verbose=0, random_state=1234, copy_x=True, n_jobs=None, algorithm='auto')
 region_metadata["clusters"]=km.fit_predict(region_metadata[["lat","lon"]].values)
 train_supplamanteary2=pd.merge(train_supplamanteary2,region_metadata, how="left", left_on=["Province_State","Country_Region"], right_on=["Province_State","Country_Region"])
 if not group_by_columns is None and len(group_by_columns)>0:
group_names=[]
for group in group_by_columns:
groupss=train_supplamanteary2[["Date", group,target1,target2]]
grp=groupss.groupby(["Date", group], as_index=False ).sum()
grp.columns=["Date", group,group +"_" + target1,group +"_" + target2 ]
train_supplamanteary2=pd.merge(train_supplamanteary2,grp, how="left", left_on=["Date", group,group], right_on=["Date", group,group])
 for gr in [group +"_" + target1,group +"_" + target2]:
rate(train_supplamanteary2, key, gr, new_target_name="rate_" +gr)
 group_names+=["rate_" +gr]<categorify>
print(f'Training final model for {epochs} epochs')
 for epoch in range(1, epochs + 1):
print(f'| epoch {epoch:03d} | train accuracy={train(model, optimizer, train_data + val_data, batch_size, token_size, log=epoch==1):.3f}' )
def get_lags(rate_array, current_index, size=20):
lag_confirmed_rate=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
lag_confirmed_rate[j]=rate_array[current_index-j]
else :
break
return lag_confirmed_rate
def days_ago_thresold_hit(full_array, indx, thresold):
days_ago_confirmed_count_10=-1
if full_array[indx]>thresold:
for j in range(indx,-1,-1):
entered=False
if full_array[j]<=thresold:
days_ago_confirmed_count_10=abs(j-indx)
 entered=True
break
if entered==False:
days_ago_confirmed_count_10=100
return days_ago_confirmed_count_10
def ewma_vectorized(data, alpha):
sums=sum([(alpha**(k+1)) *data[k] for k in range(len(data)) ])
 counts=sum([(alpha**(k+1)) for k in range(len(data)) ])
 return sums/counts
def generate_ma_std_window(rate_array, current_index, size=20, window=3):
ma_rate_confirmed=[-1 for k in range(size)]
std_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ma_rate_confirmed[j]=np.mean(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 std_rate_confirmed[j]=np.std(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 else :
break
return ma_rate_confirmed, std_rate_confirmed
def generate_ewma_window(rate_array, current_index, size=20, window=3, alpha=0.05):
ewma_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ewma_rate_confirmed[j]=ewma_vectorized(rate_array[max(0,current_index-j-window+1):current_index-j+1, ], alpha)
 else :
break
return ewma_rate_confirmed
def get_target(rate_col, indx, horizon=33, average=3, use_hard_rule=False):
target_values=[-1 for k in range(horizon)]
cou=0
for j in range(indx+1, indx+1+horizon):
if j<len(rate_col):
if average==1:
target_values[cou]=rate_col[j]
else :
if use_hard_rule and j +average <=len(rate_col):
target_values[cou]=np.mean(rate_col[j:j +average])
 else :
target_values[cou]=np.mean(rate_col[j:min(len(rate_col),j +average)])
 cou+=1
else :
break
return target_values
def get_target_count(rate_col, indx, horizon=33, average=3, use_hard_rule=False):
target_values=[-1 for k in range(horizon)]
cou=0
for j in range(indx+1, indx+1+horizon):
if j<len(rate_col):
if average==1:
target_values[cou]=rate_col[j]
else :
if use_hard_rule and j +average <=len(rate_col):
target_values[cou]=np.mean(rate_col[j:j +average])
 else :
target_values[cou]=np.mean(rate_col[j:min(len(rate_col),j +average)])
 cou+=1
else :
break
return target_values
def dereive_features(frame, confirmed, fatalities, rate_confirmed, rate_fatalities, count_confirmed, count_fatalities ,
horizon ,size=20, windows=[3,7], days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10],
extra_data=None, groups_data=None, windows_group=[3,7], size_group=20,
days_back_confimed_group=[1,10,100]):
targets=[]
if not extra_data is None:
assert len(extra_stable_columns)==extra_data.shape[1]
if not groups_data is None:
assert len(group_names)==groups_data.shape[1]
names=[]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["ma" + str(window)+ "_count_confirmed" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["ma" + str(window)+ "_count_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
if not extra_data is None:
names+=[k for k in extra_stable_columns]
if not groups_data is None:
for gg in range(groups_data.shape[1]):
for day in days_back_confimed_group:
names+=["days_ago_grooupcount_" + str(gg+1)+ "_" + str(day)]
for window in windows_group:
names+=["ma_group_" + str(gg+1)+ "_" + str(window)+ "_rate_" + str(k+1)for k in range(size_group)]
names+=["confirmed_plus" + str(k+1)for k in range(horizon)]
names+=["fatalities_plus" + str(k+1)for k in range(horizon)]
names+=["confirmed_count_plus" + str(k+1)for k in range(horizon)]
names+=["fatalities_count_plus" + str(k+1)for k in range(horizon)]
features=[]
for i in range(len(confirmed)) :
row_features=[]
lag_confirmed_rate=get_lags(rate_confirmed, i, size=size)
 for day in days_back_confimed:
days_ago_confirmed_count_10=days_ago_thresold_hit(confirmed, i, day)
 row_features+=[days_ago_confirmed_count_10]
for window in windows:
ma3_rate_confirmed,std3_rate_confirmed= generate_ma_std_window(rate_confirmed, i, size=size, window=window)
 row_features+= ma3_rate_confirmed
ma3_count_confirmed,std3_count_confirmed= generate_ma_std_window(count_confirmed, i, size=size, window=window)
 row_features+= ma3_count_confirmed
lag_fatalities_rate=get_lags(rate_fatalities, i, size=size)
 for day in days_back_fatalities:
days_ago_fatalitiescount_2=days_ago_thresold_hit(fatalities, i, day)
 row_features+=[days_ago_fatalitiescount_2]
for window in windows:
ma3_rate_fatalities,std3_rate_fatalities= generate_ma_std_window(rate_fatalities, i, size=size, window=window)
 row_features+= ma3_rate_fatalities
ma3_count_fatalities,std3_count_fatalities= generate_ma_std_window(count_fatalities, i, size=size, window=window)
 row_features+= ma3_count_fatalities
confirmed_level=0

 confirmed_level= confirmed[i]
row_features+=[confirmed_level]
fatalities_is_level=0

 fatalities_is_level= fatalities[i]
row_features+=[fatalities_is_level]
if not extra_data is None:
row_features+=extra_data[i].tolist()
if not groups_data is None:
for gg in range(groups_data.shape[1]):
this_group=groups_data[:,gg].tolist()
lag_group_rate=get_lags(this_group, i, size=size_group)
 for day in days_back_confimed_group:
days_ago_groupcount_2=days_ago_thresold_hit(this_group, i, day)
 row_features+=[days_ago_groupcount_2]
for window in windows_group:
ma3_rate_group,std3_rate_group= generate_ma_std_window(this_group, i, size=size_group, window=window)
 row_features+= ma3_rate_group
confirmed_plus=get_target(rate_confirmed, i, horizon=horizon)
 row_features+= confirmed_plus
fatalities_plus=get_target(rate_fatalities, i, horizon=horizon)
 row_features+= fatalities_plus
confirmed_plus=get_target(count_confirmed, i, horizon=horizon)
 row_features+= confirmed_plus
fatalities_plus=get_target(count_fatalities, i, horizon=horizon)
 row_features+= fatalities_plus
features.append(row_features)
 new_frame=pd.DataFrame(data=features, columns=names ).reset_index(drop=True)
 frame=frame.reset_index(drop=True)
 frame=pd.concat([frame, new_frame], axis=1)
 return frame
def feature_engineering_for_single_key(frame, group, key, horizon=33, size=20, windows=[3,7],
days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10],
extra_stable_=None, group_nams=None,windows_group=[3,7],
size_group=20, days_back_confimed_group=[1,10,100]):
mini_frame=get_data_by_key(frame, group, key, fields=None)
 mini_frame_with_features=dereive_features(mini_frame, mini_frame["ConfirmedCases"].values,
mini_frame["Fatalities"].values, mini_frame["rate_ConfirmedCases"].values,
mini_frame["rate_Fatalities"].values, mini_frame["diff_ConfirmedCases"].values,
mini_frame["diff_Fatalities"].values,
horizon ,size=size, windows=windows,
days_back_confimed=days_back_confimed, days_back_fatalities=days_back_fatalities,
extra_data=mini_frame[extra_stable_].values if not extra_stable_ is None else None,
groups_data=mini_frame[group_nams].values if not group_nams is None else None,
windows_group=windows_group, size_group=size_group,
days_back_confimed_group=days_back_confimed_group)
 return mini_frame_with_features<define_search_space>
def test(model, data, batch_size, token_size):
model.eval()
sindex = []
labels = []
with torch.no_grad() :
for batch in pool_generator(data, batch_size, token_size):
X = [torch.from_numpy(d[0])for d in batch]
X_lengths = torch.tensor([x.numel() for x in X], dtype=torch.long, device=device)
 X = torch.nn.utils.rnn.pad_sequence(X ).to(device)
 answer = model(X, X_lengths)
 label = torch.max(answer, 1)[1].cpu().numpy()
labels.append(label)
 sindex += [d[1] for d in batch]
return np.array(sindex), np.concatenate(labels )
train_frame=[]
size=10
windows=[3]
days_back_confimed=[1,5,10,20,50,100,250,500,1000]
days_back_fatalities=[1,2,5,10,20,50]
size_group=10
windows_group=[3]
days_back_confimed_group=[1,10,100]
for unique_k in tqdm(unique_keys):
mini_frame=feature_engineering_for_single_key(train, key, unique_k, horizon=horizon, size=size,
windows=windows, days_back_confimed=days_back_confimed,
days_back_fatalities=days_back_fatalities,
extra_stable_=extra_stable_columns if extra_stable_columns is not None and len(extra_stable_columns)>0 else None,
group_nams=group_names,windows_group=windows_group,
size_group=size_group, days_back_confimed_group=days_back_confimed_group
).reset_index(drop=True)
 train_frame.append(mini_frame)
 train_frame = pd.concat(train_frame, axis=0 ).reset_index(drop=True)
 new_unique_keys=train_frame['key'].unique()
for kee in new_unique_keys:
if kee not in unique_keys:
print(kee , " is not there ")
 print("==================================================")
 train_frame_supplamanteary2=None
if not train_supplamanteary2 is None:
train_frame_supplamanteary2=[]
for unique_k in tqdm(unique_keys_sup2):
mini_frame_sup=feature_engineering_for_single_key(train_supplamanteary2, key, unique_k, horizon=horizon, size=size,
windows=windows, days_back_confimed=days_back_confimed,
days_back_fatalities=days_back_fatalities,
extra_stable_=extra_stable_columns if extra_stable_columns is not None and len(extra_stable_columns)>0 else None,
group_nams=group_names,windows_group=windows_group,
size_group=size_group, days_back_confimed_group=days_back_confimed_group ).reset_index(drop=True)
 train_frame_supplamanteary2.append(mini_frame_sup)
 train_frame_supplamanteary2 = pd.concat(train_frame_supplamanteary2, axis=0 ).reset_index(drop=True)
 new_unique_keys_sup2=train_frame_supplamanteary2['key'].unique() 
<train_model>
x_test_txt = open(".. /input/x_test.txt" ).read().splitlines()
x_test_idx = [np.array([char_vocab.token2idx[c] if c in char_vocab.token2idx else unk_index for c in line])for line in x_test_txt]
test_data = [(x, idx)for idx, x in enumerate(x_test_idx)]
def bagged_set_train(X_ts,y_cs,wts, seed, estimators,xtest, xt=None,yt=None, output_name=None):
baggedpred=np.array([ 0.0 for d in range(0, xtest.shape[0])])
 for n in range(0, estimators):
params = {'objective': 'rmse',
'metric': 'rmse',
'boosting': 'gbdt',
'learning_rate': 0.005,
'drop_rate':0.01,
'skip_drop':0.6,
'uniform_drop':True,
'verbose': -1,
'num_leaves': 40,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'bagging_seed': seed + n,
'feature_fraction': 0.8,
'feature_fraction_seed': seed + n,
'min_data_in_leaf': 10,
'max_bin': 100,
'max_depth':20,
'reg_alpha':1,
'lambda_l2': 10,
'num_threads':6
}
d_train = lgb.Dataset(X_ts,y_cs, weight=wts, free_raw_data=False)
 if not type(yt)is type(None):
d_cv = lgb.Dataset(xt,yt, free_raw_data=False, reference=d_train)
 model = lgb.train(params,d_train,num_boost_round=500,
valid_sets=d_cv,
verbose_eval=50)
 else :
model = lgb.train(params,d_train,num_boost_round=500)
 preds=model.predict(xtest)
 baggedpred+=preds
if not output_name is None:
joblib.dump(( model), output_name)

baggedpred/= estimators
return baggedpred 
<predict_on_test>
index, labels = test(model, test_data, batch_size, token_size)
 order = np.argsort(index)
 labels = labels[order]
def predict(xtest,input_name=None):
baggedpred=np.array([ 0.0 for d in range(0, xtest.shape[0])])
 model= joblib.load(input_name)
 preds=model.predict(xtest)
 baggedpred+=preds
return baggedpred<define_variables>
with open('submission.csv', 'w')as f:
print('Id,Language', file=f)
 for sentence_id, lang_id in enumerate(labels):
language = lang_vocab.idx2token[lang_id]
if sentence_id < 10:
print(f'{sentence_id},{language}')
 print(f'{sentence_id},{language}', file=f )