Kaggle competition: KNIT_HACKS
data = pd.read_csv('.. /input/covid19-global-forecasting-week-2/train.csv')
 data['ConfirmedCases'] = data['ConfirmedCases'].apply(lambda x: np.log(x + 1))
data['Fatalities'] = data['Fatalities'].apply(lambda x: np.log(x + 1))
data["Country & Subdivision"] = [(data['Country_Region'][i], \
data['Province_State'][i])for i in range(len(data)) ]
unique_areas = data['Country & Subdivision'].unique()
unique_areas_dict = {j : i for i, j in enumerate(unique_areas)}
NUM_DATES = 100; TRAIN_SIZE = 57;
train = data[data['Id']%NUM_DATES <= TRAIN_SIZE]
encoded_train = np.array(list(map(lambda x: unique_areas_dict[x],
train['Country & Subdivision'].to_numpy())))\ 
.reshape(( -1, 1))
class MyModel(tf.keras.Model):
def __init__(self,):
super(MyModel, self ).__init__()
self.embedding_size = 10
self.LSTM_size = 30
self.w1 = tf.keras.layers.Embedding(len(unique_areas), self.embedding_size,)
 self.w2 = tf.keras.layers.LSTM(self.LSTM_size,
return_sequences=True,
stateful=False,)
 self.w3 = tf.keras.layers.LSTM(forecast_length,
return_sequences=True,
stateful=False, activation='relu')
 def call(self, inputs):
location, curr_num = tf.split(inputs, [-1, 1], 2)
 x = self.w1(location)
 x = tf.reshape(x, [-1,inputs.shape[1],self.embedding_size])
 x = tf.concat([x, curr_num], axis=2)
 x = self.w2(x,)
 x = self.w3(x,)
 return x
forecast_length = 13
X = np.concatenate(( encoded_train,
train['ConfirmedCases'].to_numpy().reshape(( -1, 1))),
axis=1)
 X = X.reshape(( len(unique_areas), -1, 2))
total_length = X.shape[1]
Y = X[:, :, -1]
Y = [ [y[i:i+forecast_length] for i in range(1, total_length-forecast_length+1)]
for y in Y ]
Y = np.array(Y)
 X = X[:, :-forecast_length, :]
model_1 = MyModel()
model_1.compile(optimizer='adam', loss='mae',)
 while 'history_1' not in vars() or history_1.history['loss'][-1] > 0.1:
history_1 = model_1.fit(X, Y, epochs=10000, batch_size=len(unique_areas),)
 model_1.save_weights('model_1_weights.h5')
 encoded_test = np.array(list(map(lambda x: unique_areas_dict[x],
data['Country & Subdivision'].to_numpy())))\ 
.reshape(( -1, 1))
test_X = np.concatenate(( encoded_test,
data['ConfirmedCases'].to_numpy().reshape(( -1, 1))),
axis=1)
 test_X = test_X.reshape(( len(unique_areas), -1, 2))
test_X = test_X[:, :TRAIN_SIZE, :]
subm = pd.read_csv('.. /input/covid19-global-forecasting-week-2/submission.csv')
 results = np.zeros(( X.shape[0], 0))
for i in range(4):
model_1.reset_states()
test_Y = model_1(test_X ).numpy() [:, -1, :]
results = np.concatenate(( results, np.exp(test_Y)- 1), axis=1)
 test_Y = test_Y.reshape(( -1, forecast_length, 1))
test_Y = np.concatenate(
([[[i]] * forecast_length for i in range(len(test_Y)) ], test_Y,), axis = 2)
 test_X = np.concatenate(( test_X, test_Y), axis=1)
 subm['ConfirmedCases'] = results[:, :NUM_DATES-TRAIN_SIZE].reshape(-1)
 X = np.concatenate(( encoded_train,
train['Fatalities'].to_numpy().reshape(( -1, 1))),
axis=1)
 X = X.reshape(( len(unique_areas), -1, 2))
total_length = X.shape[1]
Y = X[:, :, -1]
Y = [ [y[i:i+forecast_length] for i in range(1, total_length-forecast_length+1)]
for y in Y ]
Y = np.array(Y)
 X = X[:, :-forecast_length, :]
model_2 = MyModel()
model_2.compile(optimizer='adam', loss='mse',)
 while 'history_2' not in vars() or history_2.history['loss'][-1] > 0.01:
history_2 = model_2.fit(X, Y, epochs=10000, batch_size=len(unique_areas),)
 model_2.save_weights('model_2_weights.h5')
 encoded_test = np.array(list(map(lambda x: unique_areas_dict[x],
data['Country & Subdivision'].to_numpy())))\ 
.reshape(( -1, 1))
test_X = np.concatenate(( encoded_test,
data['Fatalities'].to_numpy().reshape(( -1, 1))),
axis=1)
 test_X = test_X.reshape(( len(unique_areas), -1, 2))
test_X = test_X[:, :TRAIN_SIZE, :]
results = np.zeros(( X.shape[0], 0))
for i in range(4):
model_2.reset_states()
test_Y = model_2(test_X ).numpy() [:, -1, :]
print(np.exp(test_Y[2])-1)
 results = np.concatenate(( results, np.exp(test_Y)- 1), axis=1)
 test_Y = test_Y.reshape(( -1, forecast_length, 1))
test_Y = np.concatenate(
([[[i]] * forecast_length for i in range(len(test_Y)) ], test_Y,), axis = 2)
 test_X = np.concatenate(( test_X, test_Y), axis=1)
 subm['Fatalities'] = results[:, :NUM_DATES-TRAIN_SIZE].reshape(-1)
 subm.to_csv('submission.csv', index=False )<set_options>
pd.set_option('display.max.columns',3000 )
warnings.filterwarnings('ignore' )<load_from_csv>
train=pd.read_csv('/kaggle/input/knit-hacks/train.csv')
 test=pd.read_csv('/kaggle/input/knit-hacks/test.csv' )
dftrain = pd.read_csv('.. /input/covid19-global-forecasting-week-2/train.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'])
 dftest = pd.read_csv('.. /input/covid19-global-forecasting-week-2/test.csv', parse_dates=['Date'] ).sort_values(by=['Country_Region', 'Date'])
 ppp_tabel = pd.read_csv('.. /input/country-ppp/Country_PPP.csv', sep='\s+')
 ppp_tabel.drop('Id', 1,inplace=True)
 ppp_tabel["Country"].replace('_',' ', regex=True,inplace=True)
 ppp_tabel["Country"].replace('United States','US', regex=True,inplace=True)
 ppp_tabel.rename(columns={'Country':'Country_Region'},inplace=True)
 ppp_tabel.sort_values('Country_Region',inplace=True)
<merge>
train['Col2'].value_counts() /train.shape[0]
dftrain['Dayofyear'] = dftrain['Date'].dt.dayofyear
dftest['Dayofyear'] = dftest['Date'].dt.dayofyear
dftest['Expo'] = dftest['Dayofyear']-89.5
dftest = dftest.merge(dftrain[['Country_Region','Province_State','Date','ConfirmedCases','Fatalities']], on=['Country_Region','Province_State','Date'], how='left', indicator=True)
 print("dftest columns =",dftest.columns)
 dftest.head(60)
<merge>
train.isnull().sum().any()
grouped=dftrain.groupby(['Country_Region','Province_State'] ).tail(8)
 grouped=grouped.groupby(['Country_Region','Province_State'] ).head(4)
 grouped.drop(['FatalityBasis'],axis=1,inplace=True)
 to_sum = ['NewCases','NewFatalities']
grouped1 = grouped.groupby(['Country_Region'])[to_sum].sum()
grouped1.rename(columns={'NewCases':'NewCases1','NewFatalities':'NewFatalities1'}, inplace=True)
 print("grouped1 columns =",grouped1.columns)
 print("grouped2 columns =",grouped2.columns)
 grouped = pd.merge(grouped1, grouped2, on=['Country_Region'])
 grouped['CasesIncreasePct'] = 100*(grouped['NewCases2']/grouped['NewCases1']-1)
 mask = grouped['CasesIncreasePct'] > 140
grouped.loc[mask,'CasesIncreasePct'] = 140
mask = grouped['CasesIncreasePct'] < 0
grouped.loc[mask,'CasesIncreasePct'] = 0
mask = grouped['CasesIncreasePct'].isnull()
grouped.loc[mask,'CasesIncreasePct'] = 0
grouped['Factor'] =(grouped['CasesIncreasePct']/100+1)**0.25
grouped = pd.merge(grouped, ppp_tabel, on=['Country_Region'])
 grouped['ppp'] = grouped['ppp']/10000.
 mask =(grouped['FatalityPct2'] > 9)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 5
mask =(grouped['FatalityPct2'] < 5)&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 5
mask =(grouped['FatalityPct2'] > 6)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 6
mask =(grouped['FatalityPct2'] < 1.5)&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 1.5
mask =(grouped['FatalityPct2'] >(9.5 - 0.43*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(9.5 - 0.43*grouped['ppp'])
 mask =(grouped['FatalityPct2'] <(5.6 - 0.5*grouped['ppp'])) &(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(5.6 - 0.5*grouped['ppp'])
 mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] <= 1)
 grouped.loc[mask,'FatalityPct2'] = 7
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] >= 7)
 grouped.loc[mask,'FatalityPct2'] = 4
mask =(grouped['FatalityPct2'].isnull())&(grouped['ppp'] > 1)&(grouped['ppp'] < 7)
 grouped.loc[mask,'FatalityPct2'] =(7.5 - 0.5*grouped['ppp'])
 grouped.tail(6)
 print("grouped columns =",grouped.columns )<merge>
cat_col=[col for col in train.columns if train[col].dtype=='O']
dftest.drop('_merge',axis=1,inplace= True)
 dftest = dftest.merge(grouped[['Country_Region','FatalityPct2','Factor']], on=['Country_Region'], how='left')
 dftest = dftest.merge(grouped_gem[['Province_State','Country_Region','ConfirmedCases_base','ConfirmedCases_init','NewCases_base','Fatalities_init','FatalityBasis']], on=['Province_State','Country_Region'], how='left')
<feature_engineering>
train=train.drop(cat_col,axis=1 )
dftest['ConfirmedCases_shift'] = dftest.groupby(['Country_Region', 'Province_State'])[['ConfirmedCases']].transform(lambda x: x.shift(1))
mask = dftest['ConfirmedCases'].isnull()
dftest.loc[mask,'NewCases'] = dftest.loc[mask,'NewCases_base']*(dftest.loc[mask,'Factor']**dftest.loc[mask,'Expo'])
 dftest['NewCases_cum'] = dftest.groupby(['Country_Region', 'Province_State'])[['NewCases']].cumsum()
dftest.loc[mask,'ConfirmedCases'] = dftest.loc[mask,'ConfirmedCases_init'] + dftest.loc[mask,'NewCases_cum']
mask3 = dftest['ConfirmedCases'] > 400000
dftest.loc[mask3,'FatalityPct2'] = dftest.loc[mask3,'FatalityPct2']*0.7
mask4 = dftest['ConfirmedCases'] > 800000
dftest.loc[mask4,'FatalityPct2'] = dftest.loc[mask4,'FatalityPct2']*0.7
dftest['FatalityBasis'] = dftest.groupby(['Country_Region', 'Province_State'])[
['ConfirmedCases']].transform(lambda x: x.shift(10))
dftest.loc[mask,'NewFatalities'] = dftest.loc[mask,'FatalityBasis'] * dftest.loc[mask,'FatalityPct2']/100
mask2 = dftest['NewFatalities'] >1000
dftest.loc[mask2,'NewFatalities'] = 1000
print("MASK2",mask2.sum())
 dftest['NewFatalities_cum'] = dftest.groupby(['Country_Region', 'Province_State'])[['NewFatalities']].cumsum()
dftest.loc[mask,'Fatalities'] = dftest.loc[mask,'Fatalities_init'] + dftest.loc[mask,'NewFatalities_cum']
print("dftest columns =",dftest.columns)
 dftest.head(5 )<drop_column>
test_new=test.copy(deep=True )
dftest.drop(['Dayofyear',
'Expo','FatalityPct2', 'Factor',
'ConfirmedCases_base', 'ConfirmedCases_init',
'NewCases_base', 'Fatalities_init', 'FatalityBasis',
'ConfirmedCases_shift',
'NewCases', 'NewCases_cum', 'NewFatalities','NewFatalities_cum'],axis=1,inplace=True)
 final = dftest.groupby(['Country_Region','Province_State'] ).tail(1)
 dftest.drop(['Province_State'],axis=1,inplace=True)
 dftest.rename(columns={'Province_State_orig':'Province_State'},inplace=True )<drop_column>
test_new=test_new.drop(cat_col,axis=1 )
dftest.drop(['Province_State','Country_Region','Date'],axis=1,inplace=True)
 print("dftest columns =",dftest.columns)
<save_to_csv>
train=train.fillna(0 )
dftest.ForecastId = dftest.ForecastId.astype('int')
 dftest['ConfirmedCases'] = dftest['ConfirmedCases'].round().astype(int)
 dftest['Fatalities'] = dftest['Fatalities'].round().astype(int)
 dftest.to_csv('submission.csv', index=False)
<set_options>
test_new=test_new.fillna(0 )
%matplotlib inline 
<load_from_csv>
! pip install -U sliced
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv")
 train.head()<feature_engineering>
train_copy=train.copy(deep=True )
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state
train_['Province_State'].fillna(EMPTY_VAL, inplace=True)
 train_['Province_State'] = train_.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1)
 test['Province_State'].fillna(EMPTY_VAL, inplace=True)
 test['Province_State'] = test.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1)
 test.head()<prepare_x_and_y>
train_copy=train_copy.drop('Col2',axis=1 )
train_['row_number'] = train_.groupby(['Country_Region', 'Province_State'] ).cumcount()
x = train_[train_["Country_Region"] == 'China'][train_["Province_State"] == 'Hubei']['row_number']
y = train_[train_["Country_Region"] == 'China'][train_["Province_State"] == 'Hubei']['ConfirmedCases']
y_ = train_[train_["Country_Region"] == 'China'][train_["Province_State"] == 'Hubei']['Fatalities']
def f(x, L, b, k, x_0):
return L /(1.+ np.exp(-k *(x - x_0)))+ b
def logistic(xs, L, k, x_0):
result = []
for x in xs:
xp = k*(x-x_0)
 if xp >= 0:
result.append(L /(1.+ np.exp(-xp)))
 else:
result.append(L * np.exp(xp)/(1.+ np.exp(xp)))
 return result
p0 = [max(y), 0.0,max(x)]
p0_ = [max(y_), 0.0,max(x)]
x_ = np.arange(0, 100, 1 ).tolist()
try:
popt, pcov = opt.curve_fit(logistic, x, y,p0)
 yfit = logistic(x_, *popt)
 popt_, pcov_ = opt.curve_fit(logistic, x, y_,p0_)
 yfit_ = logistic(x_, *popt_)
 except:
popt, pcov = opt.curve_fit(f, x, y, method="lm", maxfev=5000)
 yfit = f(x_, *popt)
 popt_, pcov_ = opt.curve_fit(f, x, y_, method="lm", maxfev=5000)
 yfit_ = f(x_, *popt_)
 fig, ax = plt.subplots(1, 1, figsize=(10, 8))
ax.plot(x, y, 'o', label ='Actual Cases')
 ax.plot(x_, yfit, '-', label ='Fitted Cases')
 ax.plot(x, y_, 'o', label ='Actual Fatalities')
 ax.plot(x_, yfit_, '-', label ='Fitted fatalities')
 ax.title.set_text('China - Hubei province')
 plt.legend(loc="center right")
 plt.show()<prepare_output>
vt=VarianceThreshold(threshold=0)
 vt_x=vt.fit_transform(train_copy )
unique = pd.DataFrame(train_.groupby(['Country_Region', 'Province_State'],as_index=False ).count())
 unique.head()<prepare_x_and_y>
vt_test=vt.transform(test_new )
def date_day_diff(d1, d2):
delta = dt.datetime.strptime(d1, "%Y-%m-%d")- dt.datetime.strptime(d2, "%Y-%m-%d")
 return delta.days
log_regions = []
for index, region in unique.iterrows() :
st = region['Province_State']
co = region['Country_Region']
rdata = train_[(train_['Province_State']==st)&(train_['Country_Region']==co)]
t = rdata['Date'].values
t = [float(date_day_diff(d, t[0])) for d in t]
y = rdata['ConfirmedCases'].values
y_ = rdata['Fatalities'].values
p0 = [max(y), 0.0, max(t)]
p0_ = [max(y_), 0.0, max(t)]
try:
popt, pcov = opt.curve_fit(logistic, t, y, p0, maxfev=10000)
 try:
popt_, pcov_ = opt.curve_fit(logistic, t, y_, p0_, maxfev=10000)
 except:
popt_, pcov_ = opt.curve_fit(f, t, y_,method="trf", maxfev=10000)
 log_regions.append(( co,st,popt,popt_))
except:
popt, pcov = opt.curve_fit(f, t, y,method="trf", maxfev=10000)
 popt_, pcov_ = opt.curve_fit(f, t, y_,method="trf", maxfev=10000)
 log_regions.append(( co,st,popt,popt_))
print("All done!" )<prepare_output>
sir = SlicedInverseRegression(n_directions=2)
log_regions = pd.DataFrame(log_regions)
 log_regions.head()<rename_columns>
sir.fit(vt_x,train['Col2'] )
log_regions.columns = ['Country_Region','Province_State','ConfirmedCases','Fatalities']
log_regions.head(1 )<define_search_space>
X_sir=sir.transform(vt_x )
log_regions['ConfirmedCases'].str[1].quantile([.1,.25,.5,.75,.95,.99] )<define_variables>
X_test=sir.transform(vt_test )
for index, rt in log_regions.iterrows() :
st = rt['Province_State']
co = rt['Country_Region']
popt = list(['ConfirmedCases'])
 popt_ = list(rt['Fatalities'])
 print(co,st,popt,popt_ )<count_duplicates>
from sklearn.linear_model import LogisticRegression
for index, rt in log_regions.iterrows() :
st = rt['Province_State']
co = rt['Country_Region']
popt = list(rt['ConfirmedCases'])
 popt_ = list(rt['Fatalities'])
 if popt_ == [0.0,0.0,69.0]:
popt_ = np.multiply(fp,popt)
 print(co,st,popt,popt_ )<concatenate>
lr=LogisticRegression().fit(X_sir,train['Col2'] )
submission = []
for index, rt in log_regions.iterrows() :
st = rt['Province_State']
co = rt['Country_Region']
popt = list(rt['ConfirmedCases'])
 popt_ = list(rt['Fatalities'])
 if popt_ == [0.0,0.0,69.0]:
popt_ = np.multiply(fp,popt)
 print(co,st,popt,popt_)
 rtest = test[(test['Province_State']==st)&(test['Country_Region']==co)]
for index, rt in rtest.iterrows() :
try:
tdate = rt['Date']
ca = logistic([date_day_diff(tdate, min(train_[(train_['Province_State']==st)&(train_['Country_Region']==co)]['Date'].values)) ], *popt)
 try:
fa = logistic([date_day_diff(tdate, min(train_[(train_['Province_State']==st)&(train_['Country_Region']==co)]['Date'].values)) ], *popt_)
 except:
fa = f([date_day_diff(tdate, min(train_[(train_['Province_State']==st)&(train_['Country_Region']==co)]['Date'].values)) ], *popt_)
 submission.append(( rt['ForecastId'], int(ca[0]), int(fa[0])))
 except:
tdate = rt['Date']
ca = f([date_day_diff(tdate, min(train_[(train_['Province_State']==st)&(train_['Country_Region']==co)]['Date'].values)) ], *popt)
 fa = f([date_day_diff(tdate, min(train_[(train_['Province_State']==st)&(train_['Country_Region']==co)]['Date'].values)) ], *popt_)
 submission.append(( rt['ForecastId'], int(ca[0]), int(fa[0])))
 print("All done!" )<save_to_csv>
submission=pd.DataFrame({'Col1':test.Col1,'Col2':lr.predict(X_test)} )
<import_modules><EOS> 
submission.to_csv('submission_reduction.csv',index=False )