Kaggle competition: GirlsGoIT competition 2020
pred_val_y, pred_test_y = train_pred(model, epochs=15 )<compute_test_metric>
data = pd.read_csv('/kaggle/input/girlsgoit-competition-2020/train.csv')
 data.head()
f1, threshold = f1_smart(val_y, pred_val_y)
 print('Optimal F1: {} at threshold: {}'.format(f1, threshold))<save_to_csv>
test = pd.read_csv('/kaggle/input/girlsgoit-competition-2020/test.csv')
 test.head()
pred_test_y =(pred_test_y >threshold ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<import_modules>
submit = pd.read_csv('/kaggle/input/girlsgoit-competition-2020/sampleSubmission.csv')
 submit.info()
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate
from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.layers import *
from keras.models import *
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.initializers import *
from keras.optimizers import *
import keras.backend as K
from keras.callbacks import *
import tensorflow as tf
import os
import time
import gc
import re
import glob<define_variables>
data[["year", "popularity"]].groupby(["year",'popularity'] ).count().max(level=0 )
max_features = 100000
maxlen = 72
embed_size = 300<define_variables>
max_per_year = data.groupby(['year'])['popularity'].apply(lambda x: x.value_counts().index[0] ).reset_index()
max_per_year
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df["question_text"] = train_df["question_text"].apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
train_df, val_df = train_test_split(train_df, test_size=0.08, random_state=2018)
 train_X = train_df["question_text"].fillna("_
val_X = val_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_

 tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)

train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]
return train_X, val_X, test_X, train_y, val_y, tokenizer.word_index<split>
toPopularity = {i:j for [i,j] in max_per_year.to_numpy() }
toPopularity[2010]
train_X, val_X, test_X, train_y, val_y, word_index = load_and_prec()<feature_engineering>
def pred_for_year(x):

 return x['year'].apply(lambda x: toPopularity[x])
 pred_for_year(data )
embedding_matrix = load_glove(word_index )<choose_model_class>
columns = [ 'acousticness', 'danceability', 'duration_ms',
'energy', 'explicit', 'instrumentalness', 'key', 'liveness', 'loudness',
'mode', 'speechiness', 'tempo', 'valence',
'year']
columns = ['year']
X = data[columns]
X.head()
def model_lstm_du(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)

x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 conc = Dense(64, activation="relu" )(conc)
 conc = Dropout(0.1 )(conc)
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model<choose_model_class>
Y = data[['popularity']]
Y.head()
def train_pred(model, epochs=2):
filepath="weights_best.h5"
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1, min_lr=0.0001, verbose=2)
 earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')
 callbacks = [checkpoint, reduce_lr]
for e in range(epochs):
model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y),callbacks=callbacks)
 model.load_weights(filepath)
 pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)
 pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)
 return pred_val_y, pred_test_y<predict_on_test>
dataX, dataY = X, Y
train_ratio = 0.70
validation_ratio = 0.15
test_ratio = 0.15
x_train, x_test, y_train, y_test = train_test_split(dataX, dataY, test_size=1 - train_ratio)
 x_val, x_test, y_val, y_test = train_test_split(x_test, y_test, test_size=test_ratio/(test_ratio + validation_ratio))
print(x_train.shape, x_val.shape, x_test.shape )
pred_val_y, pred_test_y = train_pred(model, epochs=8 )<compute_test_metric>
from sklearn.tree import DecisionTreeClassifier
f1, threshold = f1_smart(val_y, pred_val_y)
 print('Optimal F1: {} at threshold: {}'.format(f1, threshold))<save_to_csv>
plt.figure(figsize=(20, 20))
pred_test_y =(pred_test_y >threshold ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<load_from_csv>
Y_predict = pred_for_year(x_val )
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 train['CRPS']=train.Country_Region+train.Province_State.fillna('')
 test['CRPS']=test.Country_Region+test.Province_State.fillna('')
 train<feature_engineering>
print("Accuracy score:", accuracy_score(y_val, Y_predict))
print("F1 score:", f1_score(y_val, Y_predict, average='micro'))
print("Recall score:", recall_score(y_val, Y_predict, average='micro'))
train['LConfirmedCases']=np.log1p(train['ConfirmedCases'])
 train['LFatalities']=np.log1p(train['Fatalities'])
 train['LDConfirmedCases']=train.groupby('CRPS')[['LConfirmedCases']].diff()
train['LDFatalities']=train.groupby('CRPS')[['LFatalities']].diff()
train['LConfirmedCases1']=train.groupby('CRPS')[['LConfirmedCases']].shift(1)
 train['LConfirmedCases2']=train.groupby('CRPS')[['LConfirmedCases']].shift(2)
 train['LConfirmedCases3']=train.groupby('CRPS')[['LConfirmedCases']].shift(3)
 train['LConfirmedCases4']=train.groupby('CRPS')[['LConfirmedCases']].shift(4)
 train['LDConfirmedCases1']=train.groupby('CRPS')[['LDConfirmedCases']].shift(1)
 train['LDConfirmedCases2']=train.groupby('CRPS')[['LDConfirmedCases']].shift(2)
 train['LDConfirmedCases3']=train.groupby('CRPS')[['LDConfirmedCases']].shift(3)
 train['LDConfirmedCases4']=train.groupby('CRPS')[['LDConfirmedCases']].shift(4)
 train['LDConfirmedCasesMA']=(train.LDConfirmedCases+train.LDConfirmedCases1+train.LDConfirmedCases2+train.LDConfirmedCases3+train.LDConfirmedCases4)/5
train['LDConfirmedCasesMA1']=train.groupby('CRPS')[['LDConfirmedCasesMA']].shift(1)
 train['LDConfirmedCasesMA2']=train.groupby('CRPS')[['LDConfirmedCasesMA']].shift(2)
 train['LDConfirmedCasesMA3']=train.groupby('CRPS')[['LDConfirmedCasesMA']].shift(3)
 train['LDConfirmedCasesMA4']=train.groupby('CRPS')[['LDConfirmedCasesMA']].shift(4)
 train['LConfirmedCasesMA']=(train.LConfirmedCases+train.LConfirmedCases1+train.LConfirmedCases2+train.LConfirmedCases3+train.LConfirmedCases4)/5
train['LConfirmedCasesMA1']=train.groupby('CRPS')[['LConfirmedCasesMA']].shift(1)
 train['LConfirmedCasesMA2']=train.groupby('CRPS')[['LConfirmedCasesMA']].shift(2)
 train['LConfirmedCasesMA3']=train.groupby('CRPS')[['LConfirmedCasesMA']].shift(3)
 train['LConfirmedCasesMA4']=train.groupby('CRPS')[['LConfirmedCasesMA']].shift(4)
 train['LFatalities1']=train.groupby('CRPS')[['LFatalities']].shift(1)
 train['LFatalities2']=train.groupby('CRPS')[['LFatalities']].shift(2)
 train['LFatalities3']=train.groupby('CRPS')[['LFatalities']].shift(3)
 train['LFatalities4']=train.groupby('CRPS')[['LFatalities']].shift(4)
 train['LDFatalities1']=train.groupby('CRPS')[['LDFatalities']].shift(1)
 train['LDFatalities2']=train.groupby('CRPS')[['LDFatalities']].shift(2)
 train['LDFatalities3']=train.groupby('CRPS')[['LDFatalities']].shift(3)
 train['LDFatalities4']=train.groupby('CRPS')[['LDFatalities']].shift(4)
 train['LDFatalitiesMA']=(train.LDFatalities+train.LDFatalities1+train.LDFatalities2+train.LDFatalities3+train.LDFatalities4)/5
train['LDFatalitiesMA1']=train.groupby('CRPS')[['LDFatalitiesMA']].shift(1)
 train['LDFatalitiesMA2']=train.groupby('CRPS')[['LDFatalitiesMA']].shift(2)
 train['LDFatalitiesMA3']=train.groupby('CRPS')[['LDFatalitiesMA']].shift(3)
 train['LDFatalitiesMA4']=train.groupby('CRPS')[['LDFatalitiesMA']].shift(4)
 train['LFatalitiesMA']=(train.LFatalities+train.LFatalities1+train.LFatalities2+train.LFatalities3+train.LFatalities4)/5
train['LFatalitiesMA1']=train.groupby('CRPS')[['LFatalitiesMA']].shift(1)
 train['LFatalitiesMA2']=train.groupby('CRPS')[['LFatalitiesMA']].shift(2)
 train['LFatalitiesMA3']=train.groupby('CRPS')[['LFatalitiesMA']].shift(3)
 train['LFatalitiesMA4']=train.groupby('CRPS')[['LFatalitiesMA']].shift(4)
 train['serd']=train.groupby('CRPS' ).cumcount()
train.loc[train.ConfirmedCases==0,'days_since_confirmed']=0
train.loc[train.ConfirmedCases>0,'days_since_confirmed']=train[train.ConfirmedCases>0].groupby('CRPS' ).cumcount() 
<choose_model_class>
cm = confusion_matrix(y_val, Y_predict)
 cm
lgbm_cc=LGBMRegressor(num_leaves = 85,learning_rate =10**-1.89,n_estimators=100,min_sum_hessian_in_leaf=(10**-4.1),min_child_samples =2,subsample =0.97,subsample_freq=10,
colsample_bytree = 0.68,reg_lambda=10**1.4,random_state=1234,n_jobs=4)
 lgbm_f=LGBMRegressor(num_leaves = 26,learning_rate =10**-1.63,n_estimators=100,min_sum_hessian_in_leaf=(10**-4.04),min_child_samples =14,subsample =0.66,subsample_freq=5,
colsample_bytree = 0.8,reg_lambda=10**1.92,random_state=1234,n_jobs=4)
 oe=OrdinalEncoder()
X=oe.fit_transform(train[['Country_Region','Province_State']].fillna(''))
train['CR']=X[:,0]
train['PS']=X[:,1]
lgbm_cc.fit(train.loc[:,['LDConfirmedCases1','LDConfirmedCases2','LDConfirmedCases3','LDConfirmedCases4',
'LDFatalities1','LDFatalities2','LDFatalities3','LDFatalities4','days_since_confirmed','CR','PS',
'LDConfirmedCasesMA1','LDConfirmedCasesMA2','LDConfirmedCasesMA3','LDConfirmedCasesMA4',
'LDFatalitiesMA1','LDFatalitiesMA2','LDFatalitiesMA3','LDFatalitiesMA4',
'LConfirmedCases1','LConfirmedCases2','LConfirmedCases3','LConfirmedCases4',
'LFatalities1','LFatalities2','LFatalities3','LFatalities4',
'LConfirmedCasesMA1','LConfirmedCasesMA2','LConfirmedCasesMA3','LConfirmedCasesMA4',
'LFatalitiesMA1','LFatalitiesMA2','LFatalitiesMA3','LFatalitiesMA4']],train.LDConfirmedCases,categorical_feature=['CR','PS'])
 lgbm_f.fit(train.loc[:,['LDConfirmedCases1','LDConfirmedCases2','LDConfirmedCases3','LDConfirmedCases4',
'LDFatalities1','LDFatalities2','LDFatalities3','LDFatalities4','days_since_confirmed','CR','PS',
'LDConfirmedCasesMA1','LDConfirmedCasesMA2','LDConfirmedCasesMA3','LDConfirmedCasesMA4',
'LDFatalitiesMA1','LDFatalitiesMA2','LDFatalitiesMA3','LDFatalitiesMA4',
'LConfirmedCases1','LConfirmedCases2','LConfirmedCases3','LConfirmedCases4',
'LFatalities1','LFatalities2','LFatalities3','LFatalities4',
'LConfirmedCasesMA1','LConfirmedCasesMA2','LConfirmedCasesMA3','LConfirmedCasesMA4',
'LFatalitiesMA1','LFatalitiesMA2','LFatalitiesMA3','LFatalitiesMA4','LConfirmedCases','LDConfirmedCases']],train.LDFatalities,categorical_feature=['CR','PS'])
<feature_engineering>
columns = ['year' , 'acousticness', 'danceability', 'energy', 'duration_ms',
'explicit', 'instrumentalness', 'key', 'liveness', 'loudness',
'mode', 'speechiness', 'tempo', 'valence'
]
info = []
for c in range(1,4):
break
for leaf in range(3, 30,3):
for deapth in range(3, 30,3):
cs = columns[:c]
X = data[cs]
x_train, x_test, y_train, y_test = train_test_split(X, dataY, test_size=1 - train_ratio)
 x_val, x_test, y_val, y_test = train_test_split(x_test, y_test, test_size=test_ratio/(test_ratio + validation_ratio))
clf = DecisionTreeClassifier(random_state=1, max_leaf_nodes=leaf, max_depth=deapth)
 clf.fit(x_train, y_train)
 Y_predict = clf.predict(x_val)
 info.append([','.join(cs), leaf, deapth , accuracy_score(y_val, Y_predict), clf])
train['serd']=train.groupby('CRPS' ).cumcount()
trainpred = pd.concat(( train,test[test.Date>train.Date.max() ])).reset_index(drop=True)
 trainpred.sort_values(['Country_Region','Province_State','Date'],inplace=True)
 X=oe.transform(trainpred[['Country_Region','Province_State']].fillna(''))
trainpred['CR']=X[:,0]
trainpred['PS']=X[:,1]
trainpred['serd']=trainpred.groupby('CRPS' ).cumcount()
trainpred.loc[trainpred.ConfirmedCases.isnull() ,'ConfirmedCases']=1
trainpred.loc[trainpred.ConfirmedCases==0,'days_since_confirmed']=0
trainpred.loc[trainpred.ConfirmedCases>0,'days_since_confirmed']=trainpred[trainpred.ConfirmedCases>0].groupby('CRPS' ).cumcount()
trainpred['LConfirmedCases']=np.log1p(trainpred['ConfirmedCases'])
 trainpred['LFatalities']=np.log1p(trainpred['Fatalities'])
 trainpred['LDConfirmedCases']=trainpred.groupby('CRPS')[['LConfirmedCases']].diff()
trainpred['LDFatalities']=trainpred.groupby('CRPS')[['LFatalities']].diff()
for serd in range(train.serd.max() +1,trainpred.serd.max() +1):
print(serd)
 trainpred['LConfirmedCases1']=trainpred.groupby('CRPS')[['LConfirmedCases']].shift(1)
 trainpred['LConfirmedCases2']=trainpred.groupby('CRPS')[['LConfirmedCases']].shift(2)
 trainpred['LConfirmedCases3']=trainpred.groupby('CRPS')[['LConfirmedCases']].shift(3)
 trainpred['LConfirmedCases4']=trainpred.groupby('CRPS')[['LConfirmedCases']].shift(4)
 trainpred['LDConfirmedCases1']=trainpred.groupby('CRPS')[['LDConfirmedCases']].shift(1)
 trainpred['LDConfirmedCases2']=trainpred.groupby('CRPS')[['LDConfirmedCases']].shift(2)
 trainpred['LDConfirmedCases3']=trainpred.groupby('CRPS')[['LDConfirmedCases']].shift(3)
 trainpred['LDConfirmedCases4']=trainpred.groupby('CRPS')[['LDConfirmedCases']].shift(4)
 trainpred['LDConfirmedCasesMA']=(trainpred.LDConfirmedCases+trainpred.LDConfirmedCases1+trainpred.LDConfirmedCases2+trainpred.LDConfirmedCases3+trainpred.LDConfirmedCases4)/5
trainpred['LDConfirmedCasesMA1']=trainpred.groupby('CRPS')[['LDConfirmedCasesMA']].shift(1)
 trainpred['LDConfirmedCasesMA2']=trainpred.groupby('CRPS')[['LDConfirmedCasesMA']].shift(2)
 trainpred['LDConfirmedCasesMA3']=trainpred.groupby('CRPS')[['LDConfirmedCasesMA']].shift(3)
 trainpred['LDConfirmedCasesMA4']=trainpred.groupby('CRPS')[['LDConfirmedCasesMA']].shift(4)
 trainpred['LConfirmedCasesMA']=(trainpred.LConfirmedCases+trainpred.LConfirmedCases1+trainpred.LConfirmedCases2+trainpred.LConfirmedCases3+trainpred.LConfirmedCases4)/5
trainpred['LConfirmedCasesMA1']=trainpred.groupby('CRPS')[['LConfirmedCasesMA']].shift(1)
 trainpred['LConfirmedCasesMA2']=trainpred.groupby('CRPS')[['LConfirmedCasesMA']].shift(2)
 trainpred['LConfirmedCasesMA3']=trainpred.groupby('CRPS')[['LConfirmedCasesMA']].shift(3)
 trainpred['LConfirmedCasesMA4']=trainpred.groupby('CRPS')[['LConfirmedCasesMA']].shift(4)
 trainpred['LFatalities1']=trainpred.groupby('CRPS')[['LFatalities']].shift(1)
 trainpred['LFatalities2']=trainpred.groupby('CRPS')[['LFatalities']].shift(2)
 trainpred['LFatalities3']=trainpred.groupby('CRPS')[['LFatalities']].shift(3)
 trainpred['LFatalities4']=trainpred.groupby('CRPS')[['LFatalities']].shift(4)
 trainpred['LDFatalities1']=trainpred.groupby('CRPS')[['LDFatalities']].shift(1)
 trainpred['LDFatalities2']=trainpred.groupby('CRPS')[['LDFatalities']].shift(2)
 trainpred['LDFatalities3']=trainpred.groupby('CRPS')[['LDFatalities']].shift(3)
 trainpred['LDFatalities4']=trainpred.groupby('CRPS')[['LDFatalities']].shift(4)
 trainpred['LDFatalitiesMA']=(trainpred.LDFatalities+trainpred.LDFatalities1+trainpred.LDFatalities2+trainpred.LDFatalities3+trainpred.LDFatalities4)/5
trainpred['LDFatalitiesMA1']=trainpred.groupby('CRPS')[['LDFatalitiesMA']].shift(1)
 trainpred['LDFatalitiesMA2']=trainpred.groupby('CRPS')[['LDFatalitiesMA']].shift(2)
 trainpred['LDFatalitiesMA3']=trainpred.groupby('CRPS')[['LDFatalitiesMA']].shift(3)
 trainpred['LDFatalitiesMA4']=trainpred.groupby('CRPS')[['LDFatalitiesMA']].shift(4)
 trainpred['LFatalitiesMA']=(trainpred.LFatalities+trainpred.LFatalities1+trainpred.LFatalities2+trainpred.LFatalities3+trainpred.LFatalities4)/5
trainpred['LFatalitiesMA1']=trainpred.groupby('CRPS')[['LFatalitiesMA']].shift(1)
 trainpred['LFatalitiesMA2']=trainpred.groupby('CRPS')[['LFatalitiesMA']].shift(2)
 trainpred['LFatalitiesMA3']=trainpred.groupby('CRPS')[['LFatalitiesMA']].shift(3)
 trainpred['LFatalitiesMA4']=trainpred.groupby('CRPS')[['LFatalitiesMA']].shift(4)
 trainpred.loc[trainpred.serd==serd,'LDConfirmedCases']= lgbm_cc.predict(trainpred.loc[trainpred.serd==serd,['LDConfirmedCases1','LDConfirmedCases2','LDConfirmedCases3','LDConfirmedCases4',
'LDFatalities1','LDFatalities2','LDFatalities3','LDFatalities4','days_since_confirmed','CR','PS',
'LDConfirmedCasesMA1','LDConfirmedCasesMA2','LDConfirmedCasesMA3','LDConfirmedCasesMA4',
'LDFatalitiesMA1','LDFatalitiesMA2','LDFatalitiesMA3','LDFatalitiesMA4',
'LConfirmedCases1','LConfirmedCases2','LConfirmedCases3','LConfirmedCases4',
'LFatalities1','LFatalities2','LFatalities3','LFatalities4',
'LConfirmedCasesMA1','LConfirmedCasesMA2','LConfirmedCasesMA3','LConfirmedCasesMA4',
'LFatalitiesMA1','LFatalitiesMA2','LFatalitiesMA3','LFatalitiesMA4']])
 trainpred.loc[(trainpred.serd==serd)&(trainpred.LDConfirmedCases<0),'LDConfirmedCases']=0
trainpred.loc[trainpred.serd==serd,'LConfirmedCases']=trainpred.loc[trainpred.serd==serd,'LDConfirmedCases']+trainpred.loc[trainpred.serd==serd,'LConfirmedCases1']
trainpred.loc[trainpred.serd==serd,'ConfirmedCases']=np.exp(trainpred.loc[trainpred.serd==serd,'LConfirmedCases'])-1
trainpred.loc[trainpred.serd==serd,'LDFatalities']= lgbm_f.predict(trainpred.loc[trainpred.serd==serd,['LDConfirmedCases1','LDConfirmedCases2','LDConfirmedCases3','LDConfirmedCases4',
'LDFatalities1','LDFatalities2','LDFatalities3','LDFatalities4','days_since_confirmed','CR','PS',
'LDConfirmedCasesMA1','LDConfirmedCasesMA2','LDConfirmedCasesMA3','LDConfirmedCasesMA4',
'LDFatalitiesMA1','LDFatalitiesMA2','LDFatalitiesMA3','LDFatalitiesMA4',
'LConfirmedCases1','LConfirmedCases2','LConfirmedCases3','LConfirmedCases4',
'LFatalities1','LFatalities2','LFatalities3','LFatalities4',
'LConfirmedCasesMA1','LConfirmedCasesMA2','LConfirmedCasesMA3','LConfirmedCasesMA4',
'LFatalitiesMA1','LFatalitiesMA2','LFatalitiesMA3','LFatalitiesMA4','LConfirmedCases','LDConfirmedCases']])
 trainpred.loc[(trainpred.serd==serd)&(trainpred.LDFatalities<0),'LDFatalities']=0
trainpred.loc[trainpred.serd==serd,'LFatalities']=trainpred.loc[trainpred.serd==serd,'LDFatalities']+trainpred.loc[trainpred.serd==serd,'LFatalities1']
trainpred.loc[trainpred.serd==serd,'Fatalities']=np.exp(trainpred.loc[trainpred.serd==serd,'LFatalities'])-1<save_to_csv>
so = sorted(info, key=lambda x:x[-2])[-5:]
trainpred.loc[(trainpred.Date<=max(test.Date)) &(trainpred.Date>=min(test.Date)) ,'ForecastId']=test.loc[:,'ForecastId'].values
submission=trainpred.loc[trainpred.Date>=min(test.Date)][['ForecastId','ConfirmedCases','Fatalities']]
submission.ForecastId=submission.ForecastId.astype('int')
 submission.sort_values('ForecastId',inplace=True)
 submission.to_csv('submission.csv',index=False )<load_from_csv>
test_data = pd.read_csv('https://girlsgoitpublic.z6.web.core.windows.net/test.csv')
 test_data['popularity'] = pred_for_year(test_data[columns])
 test_data[['ID', 'popularity']].to_csv('submission.csv',index=False )