Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<save_to_csv>
warnings.filterwarnings("ignore")
 output_notebook(resources=INLINE )
sub_df = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df["target"] = predictions
sub_df.to_csv("submission.csv", index=False )<import_modules>
country_codes = pd.read_csv('https://raw.githubusercontent.com/plotly/datasets/master/2014_world_gdp_with_codes.csv')
 country_codes = country_codes.drop('GDP(BILLIONS)', 1)
 country_codes.rename(columns={'COUNTRY': 'Country', 'CODE': 'Code'}, inplace=True )
pyLDAvis.enable_notebook()
warnings.filterwarnings('ignore' )<load_from_csv>
virus_data = pd.read_csv('/kaggle/input/novel-corona-virus-2019-dataset/covid_19_data.csv')
 prev_index = 0
first_time = False
tmp = 0
for i, row in virus_data.iterrows() :
if(virus_data.loc[i,'SNo'] < 1342 and virus_data.loc[i,'Province/State']=='Hubei'):
if(first_time):
tmp = virus_data.loc[i,'Confirmed']
prev_index = i
virus_data.loc[i,'Confirmed'] = virus_data.loc[i,'Confirmed'] + 593
first_time = False
else:
increment = virus_data.loc[i,'Confirmed'] - tmp
tmp = virus_data.loc[i,'Confirmed']
virus_data.loc[i,'Confirmed'] = virus_data.loc[prev_index,'Confirmed'] + increment + 593
prev_index = i
virus_data.rename(columns={'Country/Region': 'Country', 'ObservationDate': 'Date'}, inplace=True)
 virus_data = virus_data.fillna('unknow')
 virus_data['Country'] = virus_data['Country'].str.replace('US','United States')
 virus_data['Country'] = virus_data['Country'].str.replace('UK','United Kingdom')
 virus_data['Country'] = virus_data['Country'].str.replace('Mainland China','China')
 virus_data['Country'] = virus_data['Country'].str.replace('South Korea','Korea, South')
 virus_data['Country'] = virus_data['Country'].str.replace('North Korea','Korea, North')
 virus_data['Country'] = virus_data['Country'].str.replace('Macau','China')
 virus_data['Country'] = virus_data['Country'].str.replace('Ivory Coast','Cote d'Ivoire')
 virus_data = pd.merge(virus_data,country_codes,on=['Country'])
sample = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')
 sample.head(3 )<load_from_csv>
top_country = virus_data.loc[virus_data['Date'] == virus_data['Date'].iloc[-1]]
top_country = top_country.groupby(['Code','Country'])['Confirmed'].sum().reset_index()
top_country = top_country.sort_values('Confirmed', ascending=False)
 top_country = top_country[:30]
top_country_codes = top_country['Country']
top_country_codes = list(top_country_codes)
 countries = virus_data[virus_data['Country'].isin(top_country_codes)]
countries_day = countries.groupby(['Date','Code','Country'])['Confirmed','Deaths','Recovered'].sum().reset_index()
exponential_line_x = []
exponential_line_y = []
for i in range(16):
exponential_line_x.append(i)
 exponential_line_y.append(i)
 china = countries_day.loc[countries_day['Code']=='CHN']
new_confirmed_cases_china = []
new_confirmed_cases_china.append(list(china['Confirmed'])[0] - list(china['Deaths'])[0]
- list(china['Recovered'])[0])
 for i in range(1,len(china)) :
new_confirmed_cases_china.append(list(china['Confirmed'])[i] -
list(china['Deaths'])[i] -
list(china['Recovered'])[i])
 italy = countries_day.loc[countries_day['Code']=='ITA']
new_confirmed_cases_ita = []
new_confirmed_cases_ita.append(list(italy['Confirmed'])[0] - list(italy['Deaths'])[0]
- list(italy['Recovered'])[0])
 for i in range(1,len(italy)) :
new_confirmed_cases_ita.append(list(italy['Confirmed'])[i] -
list(italy['Deaths'])[i] -
list(italy['Recovered'])[i])
 india = countries_day.loc[countries_day['Code']=='IND']
new_confirmed_cases_india = []
new_confirmed_cases_india.append(list(india['Confirmed'])[0] - list(india['Deaths'])[0]
- list(india['Recovered'])[0])
 for i in range(1,len(india)) :
new_confirmed_cases_india.append(list(india['Confirmed'])[i] -
list(india['Deaths'])[i] -
list(india['Recovered'])[i])
 spain = countries_day.loc[countries_day['Code']=='ESP']
new_confirmed_cases_spain = []
new_confirmed_cases_spain.append(list(spain['Confirmed'])[0] - list(spain['Deaths'])[0]
- list(spain['Recovered'])[0])
 for i in range(1,len(spain)) :
new_confirmed_cases_spain.append(list(spain['Confirmed'])[i] -
list(spain['Deaths'])[i] -
list(spain['Recovered'])[i])
 us = countries_day.loc[countries_day['Code']=='USA']
new_confirmed_cases_us = []
new_confirmed_cases_us.append(list(us['Confirmed'])[0] - list(us['Deaths'])[0]
- list(us['Recovered'])[0])
 for i in range(1,len(us)) :
new_confirmed_cases_us.append(list(us['Confirmed'])[i] -
list(us['Deaths'])[i] -
list(us['Recovered'])[i])
 german = countries_day.loc[countries_day['Code']=='DEU']
new_confirmed_cases_german = []
new_confirmed_cases_german.append(list(german['Confirmed'])[0] - list(german['Deaths'])[0]
- list(german['Recovered'])[0])
 for i in range(1,len(german)) :
new_confirmed_cases_german.append(list(german['Confirmed'])[i] -
list(german['Deaths'])[i] -
list(german['Recovered'])[i])
 p1 = figure(plot_width=800, plot_height=550, title="Trajectory of Covid-19")
 p1.grid.grid_line_alpha=0.3
p1.ygrid.band_fill_color = "olive"
p1.ygrid.band_fill_alpha = 0.1
p1.xaxis.axis_label = 'Total number of detected cases(Log scale)'
p1.yaxis.axis_label = 'New confirmed cases(Log scale)'
p1.line(exponential_line_x, exponential_line_y, line_dash="4 4", line_width=0.5)
 p1.line(np.log(list(china['Confirmed'])) , np.log(new_confirmed_cases_china), color='
legend_label='China', line_width=1)
 p1.circle(np.log(list(china['Confirmed'])[-1]), np.log(new_confirmed_cases_china[-1]), fill_color="white", size=5)
 p1.line(np.log(list(italy['Confirmed'])) , np.log(new_confirmed_cases_ita), color='
legend_label='Italy', line_width=1)
 p1.circle(np.log(list(italy['Confirmed'])[-1]), np.log(new_confirmed_cases_ita[-1]), fill_color="white", size=5)
 p1.line(np.log(list(india['Confirmed'])) , np.log(new_confirmed_cases_india), color='
legend_label='India', line_width=1)
 p1.circle(np.log(list(india['Confirmed'])[-1]), np.log(new_confirmed_cases_india[-1]), fill_color="white", size=5)
 p1.line(np.log(list(spain['Confirmed'])) , np.log(new_confirmed_cases_spain), color='
legend_label='Spain', line_width=1)
 p1.circle(np.log(list(spain['Confirmed'])[-1]), np.log(new_confirmed_cases_spain[-1]), fill_color="white", size=5)
 p1.line(np.log(list(us['Confirmed'])) , np.log(new_confirmed_cases_us), color='
legend_label='United States', line_width=1)
 p1.circle(np.log(list(us['Confirmed'])[-1]), np.log(new_confirmed_cases_us[-1]), fill_color="white", size=5)
 p1.line(np.log(list(german['Confirmed'])) , np.log(new_confirmed_cases_german), color='
legend_label='Germany', line_width=1)
 p1.circle(np.log(list(german['Confirmed'])[-1]), np.log(new_confirmed_cases_german[-1]), fill_color="white", size=5)
 p1.legend.location = "bottom_right"
output_file("coronavirus.html", title="coronavirus.py")
 show(p1 )
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv')
 train.head(3 )<load_from_csv>
countries = virus_data[virus_data['Country'].isin(top_country_codes)]
countries_day = countries.groupby(['Date','Code','Country'])['Confirmed','Deaths','Recovered'].sum().reset_index()
exponential_line_x = []
exponential_line_y = []
for i in range(16):
exponential_line_x.append(i)
 exponential_line_y.append(i)
 india = countries_day.loc[countries_day['Code']=='IND']
new_confirmed_cases_india = []
new_confirmed_cases_india.append(list(india['Confirmed'])[0] - list(india['Deaths'])[0]
- list(india['Recovered'])[0])
 for i in range(1,len(india)) :
new_confirmed_cases_india.append(list(india['Confirmed'])[i] -
list(india['Deaths'])[i] -
list(india['Recovered'])[i])
 p1 = figure(plot_width=800, plot_height=550, title="Trajectory of Covid-19 in India")
 p1.grid.grid_line_alpha=0.3
p1.ygrid.band_fill_color = "olive"
p1.ygrid.band_fill_alpha = 0.1
p1.xaxis.axis_label = 'Total number of detected cases(Log scale)'
p1.yaxis.axis_label = 'New confirmed cases(Log scale)'
p = figure(plot_width=400, plot_height=400)
 p.outline_line_width = 7
p.outline_line_alpha = 0.3
p.outline_line_color = "navy"
p1.line(exponential_line_x, exponential_line_y, line_dash="4 4", line_width=0.5)
 p1.line(np.log(list(india['Confirmed'])) , np.log(new_confirmed_cases_india), color='
legend_label='India', line_width=1)
 p1.circle(np.log(list(india['Confirmed'])[-1]), np.log(new_confirmed_cases_india[-1]), fill_color="white", size=5)
 p1.legend.location = "bottom_right"
output_file("coronavirus_india.html", title="India.py")
 show(p1 )
test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')
 test.head(3 )<define_variables>
init_notebook_mode(connected=False )
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
] 
<feature_engineering>
corona_data=pd.read_csv('/kaggle/input/novel-corona-virus-2019-dataset/covid_19_data.csv')
 choro_map=px.choropleth(corona_data,
locations="Country/Region",
locationmode = "country names",
color="Confirmed",
hover_name="Country/Region",
animation_frame="ObservationDate" 
)
 choro_map.update_layout(
 title_text = 'Global Spread of Coronavirus',
title_x = 0.5,
geo=dict(
 showframe = False,
showcoastlines = False, 
))
choro_map.show()
stopwords=stopwords.words('english')
 train['que_stopwords']=train['question_body'].apply(lambda x : [x for x in x.split() if x in stopwords])
 train['ans_stopwords']=train['answer'].apply(lambda x: [x for x in x.split() if x in stopwords] )<count_unique_values>
zone=pd.read_csv('/kaggle/input/covid-19-india-zone-classification/lockdownindiawarningzones.csv' )
def common_ngrams(col,common=10):
corpus=[]
for question in train[col].values:
words=[str(x[0]+' '+x[1])for x in ngrams(question.split() ,2)]
corpus.append(words)
 flatten=[x for one in corpus for x in one]
counter=Counter(flatten)
 most_common=counter.most_common(common)
 string,value=zip(*(most_common))
return string,value 
<set_options>
covid_India_cases = pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 covid_India_cases.rename(columns={'State/UnionTerritory': 'State', 'Cured': 'Recovered', 'Confirmed': 'Confirmed'}, inplace=True)
 statewise_cases = pd.DataFrame(covid_India_cases.groupby(['State'])['Confirmed', 'Deaths', 'Recovered'].max().reset_index())
 statewise_cases["Country"] = "India"
fig = px.treemap(statewise_cases, path=['Country','State'], values='Confirmed',
color='Confirmed', hover_data=['State'],
color_continuous_scale='Rainbow')
 fig.show()
np.set_printoptions(suppress=True )<load_from_csv>
covid_India_cases = pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 covid_India_cases.rename(columns={'State/UnionTerritory': 'State', 'Cured': 'Recovered', 'Confirmed': 'Confirmed'}, inplace=True)
 statewise_cases = pd.DataFrame(covid_India_cases.groupby(['State'])['Confirmed', 'Deaths', 'Recovered'].max().reset_index())
 last=statewise_cases
pos=pd.read_csv('.. /input/utm-of-india/UTM ZONES of INDIA.csv')
 ind_grp=last.merge(pos , left_on='State', right_on='State / Union Territory' )
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )<string_transform>
ind_map=pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 pos=pd.read_csv('.. /input/utm-of-india/UTM ZONES of INDIA.csv')
 ind_map1=ind_map.merge(pos , left_on='State/UnionTerritory', right_on='State / Union Territory')
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<compute_test_metric>
warnings.filterwarnings('ignore')
 %matplotlib inline
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.nanmean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback<split>
age_details = pd.read_csv('.. /input/covid19-in-india/AgeGroupDetails.csv')
 india_covid_19 = pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 hospital_beds = pd.read_csv('.. /input/covid19-in-india/HospitalBedsIndia.csv')
 individual_details = pd.read_csv('.. /input/covid19-in-india/IndividualDetails.csv')
 ICMR_labs = pd.read_csv('.. /input/covid19-in-india/ICMRTestingLabs.csv')
 state_testing = pd.read_csv('.. /input/covid19-in-india/StatewiseTestingDetails.csv')
gkf = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body)
 test_inputs = compute_input_arays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )<load_pretrained>
confirmed_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv')
 deaths_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_global.csv')
 recovered_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_recovered_global.csv')
 latest_data = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_daily_reports/04-04-2020.csv' )
models = []
for i in range(5):
model_path = f'.. /input/bertuned-f{i}/bertuned_f{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model)
<load_pretrained>
india_covid_19['Date'] = pd.to_datetime(india_covid_19['Date'])
 state_testing['Date'] = pd.to_datetime(state_testing['Date'] )
model_path = f'.. /input/bertf1e15/Full-0.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
dates = list(confirmed_df.columns[4:])
 dates = list(pd.to_datetime(dates))
dates_india = dates[8:]
for i in range(2):
model_path = f'.. /input/bertmodelpretrained/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
df = pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 data = df.copy()
data['Date'] = data['Date'].apply(pd.to_datetime)
 data.drop(['Sno', 'Time'],axis=1,inplace=True)
 data_apr = data[data['Date'] > pd.Timestamp(date(2020,4,12)) ]
state_cases = data_apr.groupby('State/UnionTerritory')['Confirmed','Deaths','Cured'].max().reset_index()
state_cases['Active'] = state_cases['Confirmed'] -(state_cases['Deaths']- state_cases['Cured'])
 state_cases["Death Rate(per 100)"] = np.round(100*state_cases["Deaths"]/state_cases["Confirmed"],2)
 state_cases["Cure Rate(per 100)"] = np.round(100*state_cases["Cured"]/state_cases["Confirmed"],2 )
for i in range(7):
model_path = f'.. /input/pretrained-bert/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<define_variables>
state_testing = pd.read_csv('.. /input/covid19-in-india/StatewiseTestingDetails.csv')
 state_testing
test_predictions = []<predict_on_test>
labs = pd.read_csv(".. /input/covid19-in-india/ICMRTestingLabs.csv")
 fig = px.treemap(labs, path=['state','city'],
color='city', hover_data=['lab','address'],
color_continuous_scale='reds')
 fig.show()
for model in models:
test_predictions.append(model.predict(test_inputs, batch_size=8))<prepare_output>
zone=pd.read_csv('/kaggle/input/covid-19-india-zone-classification/lockdownindiawarningzones.csv')
 zone.style.set_properties(**{'background-color': 'black',
'color': 'lawngreen',
'border-color': 'white'} )
final_predictions = np.mean(test_predictions, axis=0 )<load_pretrained>
country_codes = pd.read_csv('https://raw.githubusercontent.com/plotly/datasets/master/2014_world_gdp_with_codes.csv')
 country_codes = country_codes.drop('GDP(BILLIONS)', 1)
 country_codes.rename(columns={'COUNTRY': 'Country', 'CODE': 'Code'}, inplace=True )
n=df_test['url'].apply(lambda x:(( 'ell.stackexchange.com' in x)or('english.stackexchange.com' in x)) ).tolist()
spelling=[]
for x in n:
if x:
spelling.append(0.5)
 else:
spelling.append(0.) 
<save_to_csv>
virus_data = pd.read_csv('/kaggle/input/novel-corona-virus-2019-dataset/covid_19_data.csv')
 prev_index = 0
first_time = False
tmp = 0
for i, row in virus_data.iterrows() :
if(virus_data.loc[i,'SNo'] < 1342 and virus_data.loc[i,'Province/State']=='Hubei'):
if(first_time):
tmp = virus_data.loc[i,'Confirmed']
prev_index = i
virus_data.loc[i,'Confirmed'] = virus_data.loc[i,'Confirmed'] + 593
first_time = False
else:
increment = virus_data.loc[i,'Confirmed'] - tmp
tmp = virus_data.loc[i,'Confirmed']
virus_data.loc[i,'Confirmed'] = virus_data.loc[prev_index,'Confirmed'] + increment + 593
prev_index = i
virus_data.rename(columns={'Country/Region': 'Country', 'ObservationDate': 'Date'}, inplace=True)
 virus_data = virus_data.fillna('unknow')
 virus_data['Country'] = virus_data['Country'].str.replace('US','United States')
 virus_data['Country'] = virus_data['Country'].str.replace('UK','United Kingdom')
 virus_data['Country'] = virus_data['Country'].str.replace('Mainland China','China')
 virus_data['Country'] = virus_data['Country'].str.replace('South Korea','Korea, South')
 virus_data['Country'] = virus_data['Country'].str.replace('North Korea','Korea, North')
 virus_data['Country'] = virus_data['Country'].str.replace('Macau','China')
 virus_data['Country'] = virus_data['Country'].str.replace('Ivory Coast','Cote d'Ivoire')
 virus_data = pd.merge(virus_data,country_codes,on=['Country'])
df_sub['question_type_spelling']=spelling
df_sub.iloc[:, 1:] = final_predictions
df_sub.to_csv('submission.csv', index=False )<install_modules>
covid_India_cases = pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 covid_India_cases=covid_India_cases.dropna()
covid_India_cases.rename(columns={'State/UnionTerritory': 'State', 'Cured': 'Recovered', 'Confirmed': 'Confirmed'}, inplace=True)
 covid_India_cases = covid_India_cases.fillna('unknow')
 top_country = covid_India_cases.loc[covid_India_cases['Date'] == covid_India_cases['Date'].iloc[-1]]
top_country = top_country.groupby(['State'])['Confirmed'].sum().reset_index()
top_country = top_country.sort_values('Confirmed', ascending=False)
 top_country = top_country[:30]
top_country_codes = top_country['State']
top_country_codes = list(top_country_codes)
 countries = covid_India_cases[covid_India_cases['State'].isin(top_country_codes)]
countries_day = countries.groupby(['Date','State'])['Confirmed','Deaths','Recovered'].sum().reset_index()
exponential_line_x = []
exponential_line_y = []
for i in range(16):
exponential_line_x.append(i)
 exponential_line_y.append(i)
 Maharashtra = countries_day.loc[countries_day['State']=='Maharashtra']
Maharashtra=Maharashtra.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Maharashtra = []
new_confirmed_cases_Maharashtra.append(list(Maharashtra['Confirmed'])[0] - list(Maharashtra['Deaths'])[0]
- list(Maharashtra['Recovered'])[0])
 for i in range(1,len(Maharashtra)) :
new_confirmed_cases_Maharashtra.append(list(Maharashtra['Confirmed'])[i] -
list(Maharashtra['Deaths'])[i] -
list(Maharashtra['Recovered'])[i])
 Gujarat = countries_day.loc[countries_day['State']=='Gujarat']
Gujarat=Gujarat.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Gujarat = []
new_confirmed_cases_Gujarat.append(list(Gujarat['Confirmed'])[0] - list(Gujarat['Deaths'])[0]
- list(Gujarat['Recovered'])[0])
 for i in range(1,len(Gujarat)) :
new_confirmed_cases_Gujarat.append(list(Gujarat['Confirmed'])[i] -
list(Gujarat['Deaths'])[i] -
list(Gujarat['Recovered'])[i])
 Delhi = countries_day.loc[countries_day['State']=='Delhi']
Delhi=Delhi.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Delhi = []
new_confirmed_cases_Delhi.append(list(Delhi['Confirmed'])[0] - list(Delhi['Deaths'])[0]
- list(Delhi['Recovered'])[0])
 for i in range(1,len(Delhi)) :
new_confirmed_cases_Delhi.append(list(Delhi['Confirmed'])[i] -
list(Delhi['Deaths'])[i] -
list(Delhi['Recovered'])[i])
 Madhya_Pradesh = countries_day.loc[countries_day['State']=='Madhya Pradesh']
Madhya_Pradesh=Madhya_Pradesh.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Madhya_Pradesh = []
new_confirmed_cases_Madhya_Pradesh.append(list(Madhya_Pradesh['Confirmed'])[0] - list(Madhya_Pradesh['Deaths'])[0]
- list(Madhya_Pradesh['Recovered'])[0])
 for i in range(1,len(Madhya_Pradesh)) :
new_confirmed_cases_Madhya_Pradesh.append(list(Madhya_Pradesh['Confirmed'])[i] -
list(Madhya_Pradesh['Deaths'])[i] -
list(Madhya_Pradesh['Recovered'])[i])
 Rajasthan = countries_day.loc[countries_day['State']=='Rajasthan']
Rajasthan=Rajasthan.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Rajasthan = []
new_confirmed_cases_Rajasthan.append(list(Rajasthan['Confirmed'])[0] - list(Rajasthan['Deaths'])[0]
- list(Rajasthan['Recovered'])[0])
 for i in range(1,len(Rajasthan)) :
new_confirmed_cases_Rajasthan.append(list(Rajasthan['Confirmed'])[i] -
list(Rajasthan['Deaths'])[i] -
list(Rajasthan['Recovered'])[i])
 Uttar_Pradesh = countries_day.loc[countries_day['State']=='Uttar Pradesh']
Uttar_Pradesh=Uttar_Pradesh.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Uttar_Pradesh = []
new_confirmed_cases_Uttar_Pradesh.append(list(Uttar_Pradesh['Confirmed'])[0] - list(Uttar_Pradesh['Deaths'])[0]
- list(Uttar_Pradesh['Recovered'])[0])
 for i in range(1,len(Uttar_Pradesh)) :
new_confirmed_cases_Uttar_Pradesh.append(list(Uttar_Pradesh['Confirmed'])[i] -
list(Uttar_Pradesh['Deaths'])[i] -
list(Uttar_Pradesh['Recovered'])[i])
 Tamil_Nadu = countries_day.loc[countries_day['State']=='Tamil Nadu']
Tamil_Nadu=Tamil_Nadu.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Tamil_Nadu = []
new_confirmed_cases_Tamil_Nadu.append(list(Tamil_Nadu['Confirmed'])[0] - list(Tamil_Nadu['Deaths'])[0]
- list(Tamil_Nadu['Recovered'])[0])
 for i in range(1,len(Tamil_Nadu)) :
new_confirmed_cases_Tamil_Nadu.append(list(Tamil_Nadu['Confirmed'])[i] -
list(Tamil_Nadu['Deaths'])[i] -
list(Tamil_Nadu['Recovered'])[i])
 Andhra_Pradesh = countries_day.loc[countries_day['State']=='Andhra Pradesh']
Andhra_Pradesh=Andhra_Pradesh.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Andhra_Pradesh = []
new_confirmed_cases_Andhra_Pradesh.append(list(Andhra_Pradesh['Confirmed'])[0] - list(Andhra_Pradesh['Deaths'])[0]
- list(Andhra_Pradesh['Recovered'])[0])
 for i in range(1,len(Andhra_Pradesh)) :
new_confirmed_cases_Andhra_Pradesh.append(list(Andhra_Pradesh['Confirmed'])[i] -
list(Andhra_Pradesh['Deaths'])[i] -
list(Andhra_Pradesh['Recovered'])[i])
 West_Bengal = countries_day.loc[countries_day['State']=='West Bengal']
West_Bengal=West_Bengal.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_West_Bengal = []
new_confirmed_cases_West_Bengal.append(list(West_Bengal['Confirmed'])[0] - list(West_Bengal['Deaths'])[0]
- list(West_Bengal['Recovered'])[0])
 for i in range(1,len(West_Bengal)) :
new_confirmed_cases_West_Bengal.append(list(West_Bengal['Confirmed'])[i] -
list(West_Bengal['Deaths'])[i] -
list(West_Bengal['Recovered'])[i])
 p1 = figure(plot_width=800, plot_height=550, title="Trajectory of Covid-19")
 p1.grid.grid_line_alpha=0.3
p1.ygrid.band_fill_color = "olive"
p1.ygrid.band_fill_alpha = 0.1
p1.xaxis.axis_label = 'Total number of detected cases(Log scale)'
p1.yaxis.axis_label = 'New confirmed cases(Log scale)'
p1.line(exponential_line_x, exponential_line_y, line_dash="4 4", line_width=0.5)
 p1.line(np.log(list(Maharashtra['Confirmed'])) , np.log(new_confirmed_cases_Maharashtra), color='
legend_label='Maharashtra', line_width=1)
 p1.circle(np.log(list(Maharashtra['Confirmed'])[-1]), np.log(new_confirmed_cases_Maharashtra[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Gujarat['Confirmed'])) , np.log(new_confirmed_cases_Gujarat), color='
legend_label='Gujarat', line_width=1)
 p1.circle(np.log(list(Gujarat['Confirmed'])[-1]), np.log(new_confirmed_cases_Gujarat[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Delhi['Confirmed'])) , np.log(new_confirmed_cases_Delhi), color='
legend_label='Delhi', line_width=1)
 p1.circle(np.log(list(Delhi['Confirmed'])[-1]), np.log(new_confirmed_cases_Delhi[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Madhya_Pradesh['Confirmed'])) , np.log(new_confirmed_cases_Madhya_Pradesh), color='
legend_label='Madhya Pradesh', line_width=1)
 p1.circle(np.log(list(Madhya_Pradesh['Confirmed'])[-1]), np.log(new_confirmed_cases_Madhya_Pradesh[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Rajasthan['Confirmed'])) , np.log(new_confirmed_cases_Rajasthan), color='
legend_label='Rajasthan', line_width=1)
 p1.circle(np.log(list(Rajasthan['Confirmed'])[-1]), np.log(new_confirmed_cases_Rajasthan[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Uttar_Pradesh['Confirmed'])) , np.log(new_confirmed_cases_Uttar_Pradesh), color='
legend_label='Uttar Pradesh', line_width=1)
 p1.circle(np.log(list(Uttar_Pradesh['Confirmed'])[-1]), np.log(new_confirmed_cases_Uttar_Pradesh[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Tamil_Nadu['Confirmed'])) , np.log(new_confirmed_cases_Tamil_Nadu), color='
legend_label='Tamil Nadu', line_width=1)
 p1.circle(np.log(list(Tamil_Nadu['Confirmed'])[-1]), np.log(new_confirmed_cases_Tamil_Nadu[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Andhra_Pradesh['Confirmed'])) , np.log(new_confirmed_cases_Andhra_Pradesh), color='
legend_label='Andhra Pradesh', line_width=1)
 p1.circle(np.log(list(Andhra_Pradesh['Confirmed'])[-1]), np.log(new_confirmed_cases_Andhra_Pradesh[-1]), fill_color="white", size=5)
 p1.line(np.log(list(West_Bengal['Confirmed'])) , np.log(new_confirmed_cases_West_Bengal), color='
legend_label='West Bengal', line_width=1)
 p1.circle(np.log(list(West_Bengal['Confirmed'])[-1]), np.log(new_confirmed_cases_West_Bengal[-1]), fill_color="white", size=5)
 p1.legend.location = "bottom_right"
output_file("coronavirus.html", title="coronavirus.py")
 show(p1 )
os.system('pip install.. /input/sacremoses/sacremoses-master/ > /dev/null')
 os.system('pip install.. /input/transformers/transformers-master/ > /dev/null' )<import_modules>
plt.style.use('fivethirtyeight')
 train=pd.read_csv('/kaggle/input/coronavirus-2019ncov/covid-19-all.csv' )
BertTokenizer, BertModel, BertPreTrainedModel, BertForSequenceClassification, BertConfig,
WEIGHTS_NAME, CONFIG_NAME, AdamW, get_linear_schedule_with_warmup,
get_cosine_schedule_with_warmup, 
)
 print(transformers.__version__)
<set_options>
in_df = train[train['Country/Region']=='India'].groupby('Date')['Confirmed','Deaths','Recovered'].sum().reset_index(False)
 in_df['Active']=in_df['Confirmed']-in_df['Deaths']-in_df['Recovered']
in_df = in_df[in_df.Active>=100]
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 42
seed_everything(SEED )<load_from_csv>
in_df['day_count'] = list(range(1,len(in_df)+1))
in_df['increase'] =(in_df.Active-in_df.Active.shift(1))
in_df['rate'] =(in_df.Active-in_df.Active.shift(1)) /in_df.Active
def sigmoid(x,c,a,b):
y = c*1 /(1 + np.exp(-a*(x-b)))
 return y
xdata = np.array(list(in_df.day_count)[::2])
 ydata = np.array(list(in_df.Active)[::2])
 population=1.332*10**9
train = pd.read_csv(".. /input/google-quest-challenge/train.csv",)
 test = pd.read_csv(".. /input/google-quest-challenge/test.csv",)
 target_cols = ['question_asker_intent_understanding', 'question_body_critical',
'question_conversational', 'question_expect_short_answer',
'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written'] 
<normalization>
warnings.filterwarnings("ignore")
 output_notebook(resources=INLINE )
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)> max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length=512, t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"%(max_sequence_length,(t_new_len + a_new_len + q_new_len + 4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm_notebook(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]
def compute_output_arrays(df, columns):
return np.asarray(df[columns])
<define_variables>
country_codes = pd.read_csv('https://raw.githubusercontent.com/plotly/datasets/master/2014_world_gdp_with_codes.csv')
 country_codes = country_codes.drop('GDP(BILLIONS)', 1)
 country_codes.rename(columns={'COUNTRY': 'Country', 'CODE': 'Code'}, inplace=True )
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m' 
<categorify>
virus_data = pd.read_csv('/kaggle/input/novel-corona-virus-2019-dataset/covid_19_data.csv')
 prev_index = 0
first_time = False
tmp = 0
for i, row in virus_data.iterrows() :
if(virus_data.loc[i,'SNo'] < 1342 and virus_data.loc[i,'Province/State']=='Hubei'):
if(first_time):
tmp = virus_data.loc[i,'Confirmed']
prev_index = i
virus_data.loc[i,'Confirmed'] = virus_data.loc[i,'Confirmed'] + 593
first_time = False
else:
increment = virus_data.loc[i,'Confirmed'] - tmp
tmp = virus_data.loc[i,'Confirmed']
virus_data.loc[i,'Confirmed'] = virus_data.loc[prev_index,'Confirmed'] + increment + 593
prev_index = i
virus_data.rename(columns={'Country/Region': 'Country', 'ObservationDate': 'Date'}, inplace=True)
 virus_data = virus_data.fillna('unknow')
 virus_data['Country'] = virus_data['Country'].str.replace('US','United States')
 virus_data['Country'] = virus_data['Country'].str.replace('UK','United Kingdom')
 virus_data['Country'] = virus_data['Country'].str.replace('Mainland China','China')
 virus_data['Country'] = virus_data['Country'].str.replace('South Korea','Korea, South')
 virus_data['Country'] = virus_data['Country'].str.replace('North Korea','Korea, North')
 virus_data['Country'] = virus_data['Country'].str.replace('Macau','China')
 virus_data['Country'] = virus_data['Country'].str.replace('Ivory Coast','Cote d'Ivoire')
 virus_data = pd.merge(virus_data,country_codes,on=['Country'])
class QuestDataset(torch.utils.data.Dataset):
def __init__(self, inputs, lengths, labels = None):
self.inputs = inputs
if labels is not None:
self.labels = labels
else:
self.labels = None
self.lengths = lengths
def __getitem__(self, idx):
input_ids = self.inputs[0][idx]
input_masks = self.inputs[1][idx]
input_segments = self.inputs[2][idx]
lengths = self.lengths[idx]
if self.labels is not None:
labels = self.labels[idx]
return input_ids, input_masks, input_segments, labels, lengths
return input_ids, input_masks, input_segments, lengths
def __len__(self):
return len(self.inputs[0])
 def train_model(train_loader, optimizer, criterion, scheduler=None):
model.train()
avg_loss = 0.
 tk0 = tqdm_notebook(enumerate(train_loader))
for idx, batch in tk0:
input_ids, input_masks, input_segments, labels, _ = batch
input_ids, input_masks, input_segments, labels = input_ids.to(device), input_masks.to(device), input_segments.to(device), labels.to(device)
 output_train = model(input_ids = input_ids.long() ,
labels = None,
attention_mask = input_masks,
token_type_ids = input_segments, 
)
 logits = output_train[0]
loss = criterion(logits, labels)
 loss.backward()
optimizer.step()
if scheduler is not None:
scheduler.step()
optimizer.zero_grad()
avg_loss += loss.item() / len(train_loader)
 del input_ids, input_masks, input_segments, labels
torch.cuda.empty_cache()
gc.collect()
return avg_loss
def val_model(val_loader, val_shape, batch_size=8):
avg_val_loss = 0.
 model.eval()
valid_preds = np.zeros(( val_shape, 30))
original = np.zeros(( val_shape, 30))
tk0 = tqdm_notebook(enumerate(val_loader))
with torch.no_grad() :
for idx, batch in tk0:
input_ids, input_masks, input_segments, labels, _ = batch
input_ids, input_masks, input_segments, labels = input_ids.to(device), input_masks.to(device), input_segments.to(device), labels.to(device)
 output_val = model(input_ids = input_ids.long() ,
labels = None,
attention_mask = input_masks,
token_type_ids = input_segments, 
)
 logits = output_val[0]
avg_val_loss += criterion(logits, labels ).item() / len(val_loader)
 valid_preds[idx*batch_size :(idx+1)*batch_size] = logits.detach().cpu().squeeze().numpy()
original[idx*batch_size :(idx+1)*batch_size] = labels.detach().cpu().squeeze().numpy()
score = 0
preds = torch.sigmoid(torch.tensor(valid_preds)).numpy()
spear = []
for i in range(preds.shape[1]):
spear.append(np.nan_to_num(spearmanr(original[:, i], preds[:, i] ).correlation))
score = np.mean(spear)
 return avg_val_loss, score
def predict_result(model, test_loader, batch_size=32):
test_preds = np.zeros(( len(test), 30))
model.eval() ;
tk0 = tqdm_notebook(enumerate(test_loader))
for idx, x_batch in tk0:
with torch.no_grad() :
outputs = model(input_ids = x_batch[0].to(device),
labels = None,
attention_mask = x_batch[1].to(device),
token_type_ids = x_batch[2].to(device), 
)
 predictions = outputs[0]
test_preds[idx*batch_size :(idx+1)*batch_size] = predictions.detach().cpu().squeeze().numpy()
output = torch.sigmoid(torch.tensor(test_preds)).numpy()
return output<load_pretrained>
top_country = virus_data.loc[virus_data['Date'] == virus_data['Date'].iloc[-1]]
top_country = top_country.groupby(['Code','Country'])['Confirmed'].sum().reset_index()
top_country = top_country.sort_values('Confirmed', ascending=False)
 top_country = top_country[:30]
top_country_codes = top_country['Country']
top_country_codes = list(top_country_codes)
 countries = virus_data[virus_data['Country'].isin(top_country_codes)]
countries_day = countries.groupby(['Date','Code','Country'])['Confirmed','Deaths','Recovered'].sum().reset_index()
exponential_line_x = []
exponential_line_y = []
for i in range(16):
exponential_line_x.append(i)
 exponential_line_y.append(i)
 china = countries_day.loc[countries_day['Code']=='CHN']
new_confirmed_cases_china = []
new_confirmed_cases_china.append(list(china['Confirmed'])[0] - list(china['Deaths'])[0]
- list(china['Recovered'])[0])
 for i in range(1,len(china)) :
new_confirmed_cases_china.append(list(china['Confirmed'])[i] -
list(china['Deaths'])[i] -
list(china['Recovered'])[i])
 italy = countries_day.loc[countries_day['Code']=='ITA']
new_confirmed_cases_ita = []
new_confirmed_cases_ita.append(list(italy['Confirmed'])[0] - list(italy['Deaths'])[0]
- list(italy['Recovered'])[0])
 for i in range(1,len(italy)) :
new_confirmed_cases_ita.append(list(italy['Confirmed'])[i] -
list(italy['Deaths'])[i] -
list(italy['Recovered'])[i])
 india = countries_day.loc[countries_day['Code']=='IND']
new_confirmed_cases_india = []
new_confirmed_cases_india.append(list(india['Confirmed'])[0] - list(india['Deaths'])[0]
- list(india['Recovered'])[0])
 for i in range(1,len(india)) :
new_confirmed_cases_india.append(list(india['Confirmed'])[i] -
list(india['Deaths'])[i] -
list(india['Recovered'])[i])
 spain = countries_day.loc[countries_day['Code']=='ESP']
new_confirmed_cases_spain = []
new_confirmed_cases_spain.append(list(spain['Confirmed'])[0] - list(spain['Deaths'])[0]
- list(spain['Recovered'])[0])
 for i in range(1,len(spain)) :
new_confirmed_cases_spain.append(list(spain['Confirmed'])[i] -
list(spain['Deaths'])[i] -
list(spain['Recovered'])[i])
 us = countries_day.loc[countries_day['Code']=='USA']
new_confirmed_cases_us = []
new_confirmed_cases_us.append(list(us['Confirmed'])[0] - list(us['Deaths'])[0]
- list(us['Recovered'])[0])
 for i in range(1,len(us)) :
new_confirmed_cases_us.append(list(us['Confirmed'])[i] -
list(us['Deaths'])[i] -
list(us['Recovered'])[i])
 german = countries_day.loc[countries_day['Code']=='DEU']
new_confirmed_cases_german = []
new_confirmed_cases_german.append(list(german['Confirmed'])[0] - list(german['Deaths'])[0]
- list(german['Recovered'])[0])
 for i in range(1,len(german)) :
new_confirmed_cases_german.append(list(german['Confirmed'])[i] -
list(german['Deaths'])[i] -
list(german['Recovered'])[i])
 p1 = figure(plot_width=800, plot_height=550, title="Trajectory of Covid-19")
 p1.grid.grid_line_alpha=0.3
p1.ygrid.band_fill_color = "olive"
p1.ygrid.band_fill_alpha = 0.1
p1.xaxis.axis_label = 'Total number of detected cases(Log scale)'
p1.yaxis.axis_label = 'New confirmed cases(Log scale)'
p1.line(exponential_line_x, exponential_line_y, line_dash="4 4", line_width=0.5)
 p1.line(np.log(list(china['Confirmed'])) , np.log(new_confirmed_cases_china), color='
legend_label='China', line_width=1)
 p1.circle(np.log(list(china['Confirmed'])[-1]), np.log(new_confirmed_cases_china[-1]), fill_color="white", size=5)
 p1.line(np.log(list(italy['Confirmed'])) , np.log(new_confirmed_cases_ita), color='
legend_label='Italy', line_width=1)
 p1.circle(np.log(list(italy['Confirmed'])[-1]), np.log(new_confirmed_cases_ita[-1]), fill_color="white", size=5)
 p1.line(np.log(list(india['Confirmed'])) , np.log(new_confirmed_cases_india), color='
legend_label='India', line_width=1)
 p1.circle(np.log(list(india['Confirmed'])[-1]), np.log(new_confirmed_cases_india[-1]), fill_color="white", size=5)
 p1.line(np.log(list(spain['Confirmed'])) , np.log(new_confirmed_cases_spain), color='
legend_label='Spain', line_width=1)
 p1.circle(np.log(list(spain['Confirmed'])[-1]), np.log(new_confirmed_cases_spain[-1]), fill_color="white", size=5)
 p1.line(np.log(list(us['Confirmed'])) , np.log(new_confirmed_cases_us), color='
legend_label='United States', line_width=1)
 p1.circle(np.log(list(us['Confirmed'])[-1]), np.log(new_confirmed_cases_us[-1]), fill_color="white", size=5)
 p1.line(np.log(list(german['Confirmed'])) , np.log(new_confirmed_cases_german), color='
legend_label='Germany', line_width=1)
 p1.circle(np.log(list(german['Confirmed'])[-1]), np.log(new_confirmed_cases_german[-1]), fill_color="white", size=5)
 p1.legend.location = "bottom_right"
output_file("coronavirus.html", title="coronavirus.py")
 show(p1 )
tokenizer = BertTokenizer.from_pretrained(".. /input/pretrained-bert-models-for-pytorch/bert-base-uncased-vocab.txt")
 input_categories = list(train.columns[[1,2,5]]); input_categories
bert_model_config = '.. /input/pretrained-bert-models-for-pytorch/bert-base-uncased/bert_config.json'
bert_config = BertConfig.from_json_file(bert_model_config)
 bert_config.num_labels = 30
bert_model = 'bert-base-uncased'
do_lower_case = 'uncased' in bert_model
device = 'cuda' if torch.cuda.is_available() else 'cpu'
output_model_file = 'bert_pytorch.bin'
print(do_lower_case, bert_model, device, output_model_file )<prepare_x_and_y>
countries = virus_data[virus_data['Country'].isin(top_country_codes)]
countries_day = countries.groupby(['Date','Code','Country'])['Confirmed','Deaths','Recovered'].sum().reset_index()
exponential_line_x = []
exponential_line_y = []
for i in range(16):
exponential_line_x.append(i)
 exponential_line_y.append(i)
 india = countries_day.loc[countries_day['Code']=='IND']
new_confirmed_cases_india = []
new_confirmed_cases_india.append(list(india['Confirmed'])[0] - list(india['Deaths'])[0]
- list(india['Recovered'])[0])
 for i in range(1,len(india)) :
new_confirmed_cases_india.append(list(india['Confirmed'])[i] -
list(india['Deaths'])[i] -
list(india['Recovered'])[i])
 p1 = figure(plot_width=800, plot_height=550, title="Trajectory of Covid-19 in India")
 p1.grid.grid_line_alpha=0.3
p1.ygrid.band_fill_color = "olive"
p1.ygrid.band_fill_alpha = 0.1
p1.xaxis.axis_label = 'Total number of detected cases(Log scale)'
p1.yaxis.axis_label = 'New confirmed cases(Log scale)'
p = figure(plot_width=400, plot_height=400)
 p.outline_line_width = 7
p.outline_line_alpha = 0.3
p.outline_line_color = "navy"
p1.line(exponential_line_x, exponential_line_y, line_dash="4 4", line_width=0.5)
 p1.line(np.log(list(india['Confirmed'])) , np.log(new_confirmed_cases_india), color='
legend_label='India', line_width=1)
 p1.circle(np.log(list(india['Confirmed'])[-1]), np.log(new_confirmed_cases_india[-1]), fill_color="white", size=5)
 p1.legend.location = "bottom_right"
output_file("coronavirus_india.html", title="India.py")
 show(p1 )
NUM_FOLDS = 5
BATCH_SIZE = 8
epochs = 4
kf = GroupKFold(n_splits = NUM_FOLDS)
 scores = []
y_train = train[target_cols].values
print(bcolors.FAIL, f"For Every Fold, Train {epochs} Epochs", bcolors.ENDC)
 for fold,(train_index, val_index)in enumerate(kf.split(X=train.question_body, groups=train.question_body)) :
print(bcolors.HEADER, "Current Fold:", fold, bcolors.ENDC)
 train_df, val_df = train.iloc[train_index], train.iloc[val_index]
print("Train and Valid Shapes are", train_df.shape, val_df.shape)
 print(bcolors.HEADER, "Preparing train datasets.... ", bcolors.ENDC)
 inputs_train = compute_input_arays(train_df, input_categories, tokenizer, max_sequence_length=512)
 outputs_train = compute_output_arrays(train_df, columns = target_cols)
 outputs_train = torch.tensor(outputs_train, dtype=torch.float32)
 lengths_train = np.argmax(inputs_train[0] == 0, axis=1)
 lengths_train[lengths_train == 0] = inputs_train[0].shape[1]
print(bcolors.HEADER, "Preparing Valid datasets.... ", bcolors.ENDC)
 inputs_valid = compute_input_arays(val_df, input_categories, tokenizer, max_sequence_length=512)
 outputs_valid = compute_output_arrays(val_df, columns = target_cols)
 outputs_valid = torch.tensor(outputs_valid, dtype=torch.float32)
 lengths_valid = np.argmax(inputs_valid[0] == 0, axis=1)
 lengths_valid[lengths_valid == 0] = inputs_valid[0].shape[1]
print(bcolors.HEADER, "Preparing Dataloaders.... ", bcolors.ENDC)
 train_set = QuestDataset(inputs=inputs_train, lengths=lengths_train, labels=outputs_train)
 train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True)
 valid_set = QuestDataset(inputs=inputs_valid, lengths=lengths_valid, labels=outputs_valid)
 valid_loader = DataLoader(valid_set, batch_size=BATCH_SIZE, shuffle=False, drop_last=False)
 model = BertForSequenceClassification.from_pretrained('.. /input/pretrained-bert-models-for-pytorch/bert-base-uncased/', config=bert_config);
model.zero_grad() ;
model.to(device);
torch.cuda.empty_cache()
model.train() ;
best_score = -1.
 best_param_score = None
best_param_epoch = None
optimizer = AdamW(model.parameters() , lr=3e-5, eps=1e-7, correct_bias=False)
 criterion = nn.BCEWithLogitsLoss()
scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=0.05, num_training_steps= epochs*len(train_loader))
print("Training.... ")
 for epoch in tqdm_notebook(range(epochs)) :
torch.cuda.empty_cache()
start_time = time.time()
avg_loss = train_model(train_loader, optimizer, criterion, scheduler)
 avg_val_loss, score = val_model(valid_loader, val_shape=val_df.shape[0])
 elapsed_time = time.time() - start_time
print(bcolors.WARNING, 'Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t score={:.6f} \t time={:.2f}s'.format(
 epoch + 1, epochs, avg_loss, avg_val_loss, score, elapsed_time),
bcolors.ENDC 
)
 if best_score < score:
best_score = score
best_param_score = model.state_dict()
best_epoch_score = epoch
print(bcolors.WARNING, f'Best model came from Epoch {best_epoch_score} with score of {best_score}',bcolors.ENDC)
 model.load_state_dict(best_param_score)
 torch.save(best_param_score, 'best_param_score_{}.pt'.format(fold+1))
scores.append(best_score)
 del train_df, val_df, model, optimizer, criterion, scheduler
torch.cuda.empty_cache()
del valid_loader, train_loader, valid_set, train_set
torch.cuda.empty_cache()
torch.cuda.empty_cache()
torch.cuda.empty_cache()
gc.collect()<compute_test_metric>
init_notebook_mode(connected=False )
print(bcolors.HEADER,'Individual Fold Scores:',bcolors.ENDC)
 print(scores)
 print(bcolors.HEADER,f'QUEST BERT-base {NUM_FOLDS}-fold CV score: ',np.mean(scores),bcolors.ENDC )<install_modules>
corona_data=pd.read_csv('/kaggle/input/novel-corona-virus-2019-dataset/covid_19_data.csv')
 choro_map=px.choropleth(corona_data,
locations="Country/Region",
locationmode = "country names",
color="Confirmed",
hover_name="Country/Region",
animation_frame="ObservationDate" 
)
 choro_map.update_layout(
 title_text = 'Global Spread of Coronavirus',
title_x = 0.5,
geo=dict(
 showframe = False,
showcoastlines = False, 
))
choro_map.show()
!pip install.. /input/huggingface-transformers/sacremoses-master/sacremoses-master
!pip install.. /input/huggingface-transformers/transformers-master/transformers-master<set_options>
zone=pd.read_csv('/kaggle/input/covid-19-india-zone-classification/lockdownindiawarningzones.csv' )
tqdm.pandas()
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
covid_India_cases = pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 covid_India_cases.rename(columns={'State/UnionTerritory': 'State', 'Cured': 'Recovered', 'Confirmed': 'Confirmed'}, inplace=True)
 statewise_cases = pd.DataFrame(covid_India_cases.groupby(['State'])['Confirmed', 'Deaths', 'Recovered'].max().reset_index())
 statewise_cases["Country"] = "India"
fig = px.treemap(statewise_cases, path=['Country','State'], values='Confirmed',
color='Confirmed', hover_data=['State'],
color_continuous_scale='Rainbow')
 fig.show()
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer2 = BertTokenizer.from_pretrained(BERT_PATH+'/assets/vocab.txt', do_lower_case=True,)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv',header=0,encoding='utf-8')
 df_test = pd.read_csv(PATH+'test.csv',header=0,encoding='utf-8')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 df = pd.concat([df_train,df_test],axis=0,ignore_index=True)
 print(f)
 output_categories = ['question_asker_intent_understanding', 'question_body_critical', 'question_conversational', 'question_expect_short_answer', 'question_fact_seeking', 'question_has_commonly_accepted_answer', 'question_interestingness_others', 'question_interestingness_self', 'question_multi_intent', 'question_not_really_a_question', 'question_opinion_seeking', 'question_type_choice', 'question_type_compare', 'question_type_consequence', 'question_type_definition', 'question_type_entity', 'question_type_instructions', 'question_type_procedure', 'question_type_reason_explanation', 'question_type_spelling', 'question_well_written', 'answer_helpful', 'answer_level_of_information', 'answer_plausible', 'answer_relevance', 'answer_satisfaction', 'answer_type_instructions', 'answer_type_procedure', 'answer_type_reason_explanation', 'answer_well_written']
input_categories = ['question_title', 'question_body', 'answer']
print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )<string_transform>
covid_India_cases = pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 covid_India_cases.rename(columns={'State/UnionTerritory': 'State', 'Cured': 'Recovered', 'Confirmed': 'Confirmed'}, inplace=True)
 statewise_cases = pd.DataFrame(covid_India_cases.groupby(['State'])['Confirmed', 'Deaths', 'Recovered'].max().reset_index())
 last=statewise_cases
pos=pd.read_csv('.. /input/utm-of-india/UTM ZONES of INDIA.csv')
 ind_grp=last.merge(pos , left_on='State', right_on='State / Union Territory' )
tree_tokenizer = TreebankWordTokenizer()
def get_tree_tokens(x):
x = tree_tokenizer.tokenize(x)
 x = ' '.join(x)
 return x<feature_engineering>
ind_map=pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 pos=pd.read_csv('.. /input/utm-of-india/UTM ZONES of INDIA.csv')
 ind_map1=ind_map.merge(pos , left_on='State/UnionTerritory', right_on='State / Union Territory')
for col in input_categories:
df_train[f'treated_{col}'] = df_train[col].apply(lambda x: get_tree_tokens(x))
df_test[f'treated_{col}'] = df_test[col].apply(lambda x: get_tree_tokens(x))
df[f'treated_{col}'] = df_test[col].apply(lambda x: get_tree_tokens(x))<choose_model_class>
warnings.filterwarnings('ignore')
 %matplotlib inline
tokenizer = text.Tokenizer(lower=False )<prepare_x_and_y>
age_details = pd.read_csv('.. /input/covid19-in-india/AgeGroupDetails.csv')
 india_covid_19 = pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 hospital_beds = pd.read_csv('.. /input/covid19-in-india/HospitalBedsIndia.csv')
 individual_details = pd.read_csv('.. /input/covid19-in-india/IndividualDetails.csv')
 ICMR_labs = pd.read_csv('.. /input/covid19-in-india/ICMRTestingLabs.csv')
 state_testing = pd.read_csv('.. /input/covid19-in-india/StatewiseTestingDetails.csv')
X_train_question = df_train['question_body']
X_train_title = df_train['question_title']
X_train_answer = df_train['answer']
X_test_question = df_test['question_body']
X_test_title = df_test['question_title']
X_test_answer = df_test['answer']<train_model>
confirmed_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv')
 deaths_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_global.csv')
 recovered_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_recovered_global.csv')
 latest_data = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_daily_reports/04-04-2020.csv' )
tokenizer.fit_on_texts(list(X_train_title)+list(X_train_question)+list(X_train_answer)+list(X_test_title)+list(X_test_question)+list(X_test_answer))<choose_model_class>
india_covid_19['Date'] = pd.to_datetime(india_covid_19['Date'])
 state_testing['Date'] = pd.to_datetime(state_testing['Date'] )
nlp = English()
sentencizer = nlp.create_pipe('sentencizer')
 nlp.add_pipe(sentencizer )<string_transform>
dates = list(confirmed_df.columns[4:])
 dates = list(pd.to_datetime(dates))
dates_india = dates[8:]
def split_document(texts):
all_sents = []
max_num_sentences = 0.0
for text in texts:
doc = nlp(text)
 sents=[]
for i,sent in enumerate(doc.sents):
sents.append(sent.text)
 all_sents.append(sents)
 return all_sents
X_train_question = split_document(X_train_question)
 X_train_answer = split_document(X_train_answer)
 X_test_question = split_document(X_test_question)
 X_test_answer = split_document(X_test_answer)
<feature_engineering>
df = pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 data = df.copy()
data['Date'] = data['Date'].apply(pd.to_datetime)
 data.drop(['Sno', 'Time'],axis=1,inplace=True)
 data_apr = data[data['Date'] > pd.Timestamp(date(2020,4,12)) ]
state_cases = data_apr.groupby('State/UnionTerritory')['Confirmed','Deaths','Cured'].max().reset_index()
state_cases['Active'] = state_cases['Confirmed'] -(state_cases['Deaths']- state_cases['Cured'])
 state_cases["Death Rate(per 100)"] = np.round(100*state_cases["Deaths"]/state_cases["Confirmed"],2)
 state_cases["Cure Rate(per 100)"] = np.round(100*state_cases["Cured"]/state_cases["Confirmed"],2 )
def add_question_metadata_features(text):
doc=nlp(text)
 indirect = 0
choice_words=0
reason_explanation_words = 0
question_count = 0
for sent in doc.sents:
if '?' in sent.text and '?' == sent.text[-1]:
question_count += 1
for token in sent:
if token.text.lower() =='why':
reason_explanation_words+=1
elif token.text.lower() =='or':
choice_words+=1
if question_count==0:
indirect+=1
return np.array([indirect, question_count, reason_explanation_words, choice_words] )<count_duplicates>
state_testing = pd.read_csv('.. /input/covid19-in-india/StatewiseTestingDetails.csv')
 state_testing
ans_user_and_category=df_train[df_train[['answer_user_name', 'category']].duplicated() ][['answer_user_name', 'category']].values
ans_user_and_category.shape<string_transform>
labs = pd.read_csv(".. /input/covid19-in-india/ICMRTestingLabs.csv")
 fig = px.treemap(labs, path=['state','city'],
color='city', hover_data=['lab','address'],
color_continuous_scale='reds')
 fig.show()
def question_answer_author_same(df):
q_username = df['question_user_name']
a_username = df['answer_user_name']
author_same=[]
for i in range(len(df)) :
if q_username[i] == a_username[i]:
author_same.append(int(1))
else:
author_same.append(int(0))
return author_same 
<feature_engineering>
zone=pd.read_csv('/kaggle/input/covid-19-india-zone-classification/lockdownindiawarningzones.csv')
 zone.style.set_properties(**{'background-color': 'black',
'color': 'lawngreen',
'border-color': 'white'} )
def add_external_features(df):
df['question_body'] = df['question_body'].progress_apply(lambda x: str(x))
df['question_body_num_words'] = df['question_body'].str.count('\S+')
 df['answer'] = df['answer'].progress_apply(lambda x: str(x))
df['answer_num_words'] = df['answer'].str.count('\S+')
 df['question_vs_answer_length'] = df['question_body_num_words']/df['answer_num_words']
df['q_a_author_same'] = question_answer_author_same(df)
 answer_user_cat = []
for i in tqdm(df[['answer_user_name', 'category']].values):
if i in ans_user_and_category:
answer_user_cat.append(int(1))
else:
answer_user_cat.append(int(0))
df['answer_user_cat'] = answer_user_cat
handmade_features=[]
for text in df['question_body'].values:
handmade_features.append(add_question_metadata_features(text))
return df, np.array(handmade_features)
<create_dataframe>
country_codes = pd.read_csv('https://raw.githubusercontent.com/plotly/datasets/master/2014_world_gdp_with_codes.csv')
 country_codes = country_codes.drop('GDP(BILLIONS)', 1)
 country_codes.rename(columns={'COUNTRY': 'Country', 'CODE': 'Code'}, inplace=True )
df_train, handmade_features = add_external_features(df_train)
 df_test, handmade_features_test = add_external_features(df_test)
 df_train = pd.concat([df_train,pd.DataFrame(handmade_features, columns=['indirect', 'question_count', 'reason_explanation_words', 'choice_words'])],axis=1)
 df_test = pd.concat([df_test,pd.DataFrame(handmade_features_test, columns=['indirect', 'question_count', 'reason_explanation_words', 'choice_words'])],axis=1 )<normalization>
virus_data = pd.read_csv('/kaggle/input/novel-corona-virus-2019-dataset/covid_19_data.csv')
 prev_index = 0
first_time = False
tmp = 0
for i, row in virus_data.iterrows() :
if(virus_data.loc[i,'SNo'] < 1342 and virus_data.loc[i,'Province/State']=='Hubei'):
if(first_time):
tmp = virus_data.loc[i,'Confirmed']
prev_index = i
virus_data.loc[i,'Confirmed'] = virus_data.loc[i,'Confirmed'] + 593
first_time = False
else:
increment = virus_data.loc[i,'Confirmed'] - tmp
tmp = virus_data.loc[i,'Confirmed']
virus_data.loc[i,'Confirmed'] = virus_data.loc[prev_index,'Confirmed'] + increment + 593
prev_index = i
virus_data.rename(columns={'Country/Region': 'Country', 'ObservationDate': 'Date'}, inplace=True)
 virus_data = virus_data.fillna('unknow')
 virus_data['Country'] = virus_data['Country'].str.replace('US','United States')
 virus_data['Country'] = virus_data['Country'].str.replace('UK','United Kingdom')
 virus_data['Country'] = virus_data['Country'].str.replace('Mainland China','China')
 virus_data['Country'] = virus_data['Country'].str.replace('South Korea','Korea, South')
 virus_data['Country'] = virus_data['Country'].str.replace('North Korea','Korea, North')
 virus_data['Country'] = virus_data['Country'].str.replace('Macau','China')
 virus_data['Country'] = virus_data['Country'].str.replace('Ivory Coast','Cote d'Ivoire')
 virus_data = pd.merge(virus_data,country_codes,on=['Country'])
num_words_scaler = MinMaxScaler()
df_train[['question_body_num_words', 'answer_num_words']] = num_words_scaler.fit_transform(df_train[['question_body_num_words', 'answer_num_words']].values)
 df_test[['question_body_num_words', 'answer_num_words']] = num_words_scaler.transform(df_test[['question_body_num_words', 'answer_num_words']].values )<categorify>
covid_India_cases = pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 covid_India_cases=covid_India_cases.dropna()
covid_India_cases.rename(columns={'State/UnionTerritory': 'State', 'Cured': 'Recovered', 'Confirmed': 'Confirmed'}, inplace=True)
 covid_India_cases = covid_India_cases.fillna('unknow')
 top_country = covid_India_cases.loc[covid_India_cases['Date'] == covid_India_cases['Date'].iloc[-1]]
top_country = top_country.groupby(['State'])['Confirmed'].sum().reset_index()
top_country = top_country.sort_values('Confirmed', ascending=False)
 top_country = top_country[:30]
top_country_codes = top_country['State']
top_country_codes = list(top_country_codes)
 countries = covid_India_cases[covid_India_cases['State'].isin(top_country_codes)]
countries_day = countries.groupby(['Date','State'])['Confirmed','Deaths','Recovered'].sum().reset_index()
exponential_line_x = []
exponential_line_y = []
for i in range(16):
exponential_line_x.append(i)
 exponential_line_y.append(i)
 Maharashtra = countries_day.loc[countries_day['State']=='Maharashtra']
Maharashtra=Maharashtra.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Maharashtra = []
new_confirmed_cases_Maharashtra.append(list(Maharashtra['Confirmed'])[0] - list(Maharashtra['Deaths'])[0]
- list(Maharashtra['Recovered'])[0])
 for i in range(1,len(Maharashtra)) :
new_confirmed_cases_Maharashtra.append(list(Maharashtra['Confirmed'])[i] -
list(Maharashtra['Deaths'])[i] -
list(Maharashtra['Recovered'])[i])
 Gujarat = countries_day.loc[countries_day['State']=='Gujarat']
Gujarat=Gujarat.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Gujarat = []
new_confirmed_cases_Gujarat.append(list(Gujarat['Confirmed'])[0] - list(Gujarat['Deaths'])[0]
- list(Gujarat['Recovered'])[0])
 for i in range(1,len(Gujarat)) :
new_confirmed_cases_Gujarat.append(list(Gujarat['Confirmed'])[i] -
list(Gujarat['Deaths'])[i] -
list(Gujarat['Recovered'])[i])
 Delhi = countries_day.loc[countries_day['State']=='Delhi']
Delhi=Delhi.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Delhi = []
new_confirmed_cases_Delhi.append(list(Delhi['Confirmed'])[0] - list(Delhi['Deaths'])[0]
- list(Delhi['Recovered'])[0])
 for i in range(1,len(Delhi)) :
new_confirmed_cases_Delhi.append(list(Delhi['Confirmed'])[i] -
list(Delhi['Deaths'])[i] -
list(Delhi['Recovered'])[i])
 Madhya_Pradesh = countries_day.loc[countries_day['State']=='Madhya Pradesh']
Madhya_Pradesh=Madhya_Pradesh.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Madhya_Pradesh = []
new_confirmed_cases_Madhya_Pradesh.append(list(Madhya_Pradesh['Confirmed'])[0] - list(Madhya_Pradesh['Deaths'])[0]
- list(Madhya_Pradesh['Recovered'])[0])
 for i in range(1,len(Madhya_Pradesh)) :
new_confirmed_cases_Madhya_Pradesh.append(list(Madhya_Pradesh['Confirmed'])[i] -
list(Madhya_Pradesh['Deaths'])[i] -
list(Madhya_Pradesh['Recovered'])[i])
 Rajasthan = countries_day.loc[countries_day['State']=='Rajasthan']
Rajasthan=Rajasthan.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Rajasthan = []
new_confirmed_cases_Rajasthan.append(list(Rajasthan['Confirmed'])[0] - list(Rajasthan['Deaths'])[0]
- list(Rajasthan['Recovered'])[0])
 for i in range(1,len(Rajasthan)) :
new_confirmed_cases_Rajasthan.append(list(Rajasthan['Confirmed'])[i] -
list(Rajasthan['Deaths'])[i] -
list(Rajasthan['Recovered'])[i])
 Uttar_Pradesh = countries_day.loc[countries_day['State']=='Uttar Pradesh']
Uttar_Pradesh=Uttar_Pradesh.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Uttar_Pradesh = []
new_confirmed_cases_Uttar_Pradesh.append(list(Uttar_Pradesh['Confirmed'])[0] - list(Uttar_Pradesh['Deaths'])[0]
- list(Uttar_Pradesh['Recovered'])[0])
 for i in range(1,len(Uttar_Pradesh)) :
new_confirmed_cases_Uttar_Pradesh.append(list(Uttar_Pradesh['Confirmed'])[i] -
list(Uttar_Pradesh['Deaths'])[i] -
list(Uttar_Pradesh['Recovered'])[i])
 Tamil_Nadu = countries_day.loc[countries_day['State']=='Tamil Nadu']
Tamil_Nadu=Tamil_Nadu.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Tamil_Nadu = []
new_confirmed_cases_Tamil_Nadu.append(list(Tamil_Nadu['Confirmed'])[0] - list(Tamil_Nadu['Deaths'])[0]
- list(Tamil_Nadu['Recovered'])[0])
 for i in range(1,len(Tamil_Nadu)) :
new_confirmed_cases_Tamil_Nadu.append(list(Tamil_Nadu['Confirmed'])[i] -
list(Tamil_Nadu['Deaths'])[i] -
list(Tamil_Nadu['Recovered'])[i])
 Andhra_Pradesh = countries_day.loc[countries_day['State']=='Andhra Pradesh']
Andhra_Pradesh=Andhra_Pradesh.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Andhra_Pradesh = []
new_confirmed_cases_Andhra_Pradesh.append(list(Andhra_Pradesh['Confirmed'])[0] - list(Andhra_Pradesh['Deaths'])[0]
- list(Andhra_Pradesh['Recovered'])[0])
 for i in range(1,len(Andhra_Pradesh)) :
new_confirmed_cases_Andhra_Pradesh.append(list(Andhra_Pradesh['Confirmed'])[i] -
list(Andhra_Pradesh['Deaths'])[i] -
list(Andhra_Pradesh['Recovered'])[i])
 West_Bengal = countries_day.loc[countries_day['State']=='West Bengal']
West_Bengal=West_Bengal.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_West_Bengal = []
new_confirmed_cases_West_Bengal.append(list(West_Bengal['Confirmed'])[0] - list(West_Bengal['Deaths'])[0]
- list(West_Bengal['Recovered'])[0])
 for i in range(1,len(West_Bengal)) :
new_confirmed_cases_West_Bengal.append(list(West_Bengal['Confirmed'])[i] -
list(West_Bengal['Deaths'])[i] -
list(West_Bengal['Recovered'])[i])
 p1 = figure(plot_width=800, plot_height=550, title="Trajectory of Covid-19")
 p1.grid.grid_line_alpha=0.3
p1.ygrid.band_fill_color = "olive"
p1.ygrid.band_fill_alpha = 0.1
p1.xaxis.axis_label = 'Total number of detected cases(Log scale)'
p1.yaxis.axis_label = 'New confirmed cases(Log scale)'
p1.line(exponential_line_x, exponential_line_y, line_dash="4 4", line_width=0.5)
 p1.line(np.log(list(Maharashtra['Confirmed'])) , np.log(new_confirmed_cases_Maharashtra), color='
legend_label='Maharashtra', line_width=1)
 p1.circle(np.log(list(Maharashtra['Confirmed'])[-1]), np.log(new_confirmed_cases_Maharashtra[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Gujarat['Confirmed'])) , np.log(new_confirmed_cases_Gujarat), color='
legend_label='Gujarat', line_width=1)
 p1.circle(np.log(list(Gujarat['Confirmed'])[-1]), np.log(new_confirmed_cases_Gujarat[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Delhi['Confirmed'])) , np.log(new_confirmed_cases_Delhi), color='
legend_label='Delhi', line_width=1)
 p1.circle(np.log(list(Delhi['Confirmed'])[-1]), np.log(new_confirmed_cases_Delhi[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Madhya_Pradesh['Confirmed'])) , np.log(new_confirmed_cases_Madhya_Pradesh), color='
legend_label='Madhya Pradesh', line_width=1)
 p1.circle(np.log(list(Madhya_Pradesh['Confirmed'])[-1]), np.log(new_confirmed_cases_Madhya_Pradesh[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Rajasthan['Confirmed'])) , np.log(new_confirmed_cases_Rajasthan), color='
legend_label='Rajasthan', line_width=1)
 p1.circle(np.log(list(Rajasthan['Confirmed'])[-1]), np.log(new_confirmed_cases_Rajasthan[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Uttar_Pradesh['Confirmed'])) , np.log(new_confirmed_cases_Uttar_Pradesh), color='
legend_label='Uttar Pradesh', line_width=1)
 p1.circle(np.log(list(Uttar_Pradesh['Confirmed'])[-1]), np.log(new_confirmed_cases_Uttar_Pradesh[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Tamil_Nadu['Confirmed'])) , np.log(new_confirmed_cases_Tamil_Nadu), color='
legend_label='Tamil Nadu', line_width=1)
 p1.circle(np.log(list(Tamil_Nadu['Confirmed'])[-1]), np.log(new_confirmed_cases_Tamil_Nadu[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Andhra_Pradesh['Confirmed'])) , np.log(new_confirmed_cases_Andhra_Pradesh), color='
legend_label='Andhra Pradesh', line_width=1)
 p1.circle(np.log(list(Andhra_Pradesh['Confirmed'])[-1]), np.log(new_confirmed_cases_Andhra_Pradesh[-1]), fill_color="white", size=5)
 p1.line(np.log(list(West_Bengal['Confirmed'])) , np.log(new_confirmed_cases_West_Bengal), color='
legend_label='West Bengal', line_width=1)
 p1.circle(np.log(list(West_Bengal['Confirmed'])[-1]), np.log(new_confirmed_cases_West_Bengal[-1]), fill_color="white", size=5)
 p1.legend.location = "bottom_right"
output_file("coronavirus.html", title="coronavirus.py")
 show(p1 )
df=pd.concat([df,pd.get_dummies(df['host'], drop_first=False, prefix='host')],axis=1)
 df=pd.concat([df,pd.get_dummies(df['category'], drop_first=False, prefix='cat')],axis=1 )<define_variables>
plt.style.use('fivethirtyeight')
 train=pd.read_csv('/kaggle/input/coronavirus-2019ncov/covid-19-all.csv' )
len(['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')] )<merge>
in_df = train[train['Country/Region']=='India'].groupby('Date')['Confirmed','Deaths','Recovered'].sum().reset_index(False)
 in_df['Active']=in_df['Confirmed']-in_df['Deaths']-in_df['Recovered']
in_df = in_df[in_df.Active>=100]
<categorify><EOS> 
in_df['day_count'] = list(range(1,len(in_df)+1))
in_df['increase'] =(in_df.Active-in_df.Active.shift(1))
in_df['rate'] =(in_df.Active-in_df.Active.shift(1)) /in_df.Active
def sigmoid(x,c,a,b):
y = c*1 /(1 + np.exp(-a*(x-b)))
 return y
xdata = np.array(list(in_df.day_count)[::2])
 ydata = np.array(list(in_df.Active)[::2])
 population=1.332*10**9