Kaggle competition: COVID19 Global Forecasting (Week 2)
train_df["price_t"] =train_df["price"]/train_df["bedrooms"]
test_df["price_t"] = test_df["price"]/test_df["bedrooms"]
train_df["room_sum"] = train_df["bedrooms"]+train_df["bathrooms"]
test_df["room_sum"] = test_df["bedrooms"]+test_df["bathrooms"]
train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 features_to_use=["bathrooms", "bedrooms", "latitude", "longitude", "price","price_t","num_photos", "num_features", "num_description_words","listing_id"]<train_model>
path = '.. /input/covid19-global-forecasting-week-2/'
train = pd.read_csv(path + 'train.csv')
 column_name_map = {
'Country_Region' : 'country',
'Province_State' : 'state',
'Date' : 'date',
'ConfirmedCases' : 'cases',
'Fatalities' : 'deaths',
}
train = train.rename(columns = column_name_map)
 from_date = train['date'].min()
to_date = train['date'].max()
train['state'].fillna('-', inplace = True)
 print(train.dtypes )
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0, num_rounds=1000):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.1
param['max_depth'] = 6
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y, model<categorify>
path = '.. /input/covid19-global-forecasting-week-2/'
test = pd.read_csv(path + 'test.csv')
 column_name_map = {
'Country_Region' : 'country',
'Province_State' : 'state',
'Date' : 'date',
'ForecastId' : 'id',
}
test = test.rename(columns = column_name_map)
 test['state'].fillna('-', inplace = True)
 regions = test[['state', 'country']].drop_duplicates()
regions.head()
categorical = ["display_address", "manager_id", "building_id", "street_address"]
for f in categorical:
if train_df[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[f].values)+ list(test_df[f].values))
train_df[f] = lbl.transform(list(train_df[f].values))
test_df[f] = lbl.transform(list(test_df[f].values))
features_to_use.append(f )<feature_engineering>
def logistic(xs, l, L, k, x_0):
result = []
for x in xs:
xp = k*(x-x_0)
 if xp >= 0:
result.append(l +(L-l)/(1.+ np.exp(-xp)))
 else:
result.append(l +(L-l)* np.exp(xp)/(1.+ np.exp(xp)))
 return result
def date_day_diff(d1, d2):
delta = dt.datetime.strptime(d1, "%Y-%m-%d")- dt.datetime.strptime(d2, "%Y-%m-%d")
 return delta.days
train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
print(train_df["features"].head())
 tfidf = CountVectorizer(stop_words='english', max_features=200)
 tr_sparse = tfidf.fit_transform(train_df["features"])
 te_sparse = tfidf.transform(test_df["features"] )<prepare_x_and_y>
predictions_cases = []
plot = True
for index, region in regions.iterrows() :
predicted = False
st = region['state']
co = region['country']
rdata = train[(train['state']==st)&(train['country']==co)]
rtest = test[(test['state']==st)&(test['country']==co)]
window = rdata[rdata['cases']>=100]['date']
if(window.count() < 10):
window = rdata[rdata['cases']>=10]['date']
if(window.count() >= 10):
start_date = window.min()
rdata = rdata[rdata['date']>=start_date]
t = rdata['date'].values
t = [float(date_day_diff(d, start_date)) for d in t]
y = rdata['cases'].values
try:
bounds =([-1e6, -1e6, 0.001, 0.0], [1e6, 1e6, 0.999, t[-1]+10])
 popt, pcov = curve_fit(logistic, t, y, bounds = bounds)
 residuals = y - logistic(t, *popt)
 ss_res = np.sum(residuals**2)
 ss_tot = np.sum(( y - np.mean(y)) **2)
 rs = 1 -(ss_res / ss_tot)
 if plot:
print(st, co)
 print(popt)
 print('R squared: ', rs)
 T = np.arange(0, 60, 1 ).tolist()
yfit = logistic(T, *popt)
 pylab.plot(t, y, 'o')
 pylab.plot(T, yfit)
 pylab.show()
if rs>=0.95:
for index, rt in rtest.iterrows() :
tdate = rt['date']
prev_max = 0
if(tdate<=to_date):
ca = list(train[(train['date']==tdate)&(train['state']==st)&(train['country']==co)]['cases'].values)[0]
prev_max = max(prev_max, ca)
 else:
ca = logistic([date_day_diff(tdate, start_date)], *popt)
 ca = max(prev_max, ca[0])
 prev_max = ca
predictions_cases.append(( rt['id'], int(ca)))
 predicted = True
except:
pass
if not predicted:
t = rdata['date'].values
start_date = t[0]
t = np.array([float(date_day_diff(d, start_date)) for d in t])
 y = rdata['cases'].values
linreg = LinearRegression()
linreg.fit(t[-10:].reshape(-1, 1), y[-10:])
 m = linreg.coef_[0]
b = linreg.intercept_
if plot:
print(st, co)
 print(linreg.intercept_, linreg.coef_)
 T = np.arange(0, 90, 1 ).tolist()
y_pred = [m*x+b for x in T]
pylab.plot(t, y, 'o')
 pylab.plot(T, y_pred)
 pylab.show()
for index, rt in rtest.iterrows() :
tdate = rt['date']
prev_max = 0
if(tdate<=to_date):
ca = list(train[(train['date']==tdate)&(train['state']==st)&(train['country']==co)]['cases'].values)[0]
prev_max = max(prev_max, ca)
 else:
ca = m*date_day_diff(tdate, start_date)+ b
ca = max(ca, prev_max)
 prev_max = ca
predictions_cases.append(( rt['id'], int(ca)) )
train_X = sparse.hstack([train_df[features_to_use], tr_sparse] ).tocsr()
test_X = sparse.hstack([test_df[features_to_use], te_sparse] ).tocsr()
target_num_map = {'high':0, 'medium':1, 'low':2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))
print(train_X.shape, test_X.shape )<compute_train_metric>
data = {
'ForecastId': [pred[0] for pred in predictions_cases],
'ConfirmedCases': [pred[1] for pred in predictions_cases],
}
df_cases = pd.DataFrame(data, columns = data.keys())
 df_cases.head()
cv_scores = []
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2016)
 for dev_index, val_index in kf.split(range(train_X.shape[0])) :
dev_X, val_X = train_X[dev_index,:], train_X[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
preds, model = runXGB(dev_X, dev_y, val_X, val_y)
 cv_scores.append(log_loss(val_y, preds))
print(cv_scores)
 break<save_to_csv>
predictions_deaths = []
plot = True
for index, region in regions.iterrows() :
predicted = False
st = region['state']
co = region['country']
rdata = train[(train['state']==st)&(train['country']==co)]
rtest = test[(test['state']==st)&(test['country']==co)]
window = rdata[rdata['deaths']>=100]['date']
if(window.count() < 10):
window = rdata[rdata['deaths']>=10]['date']
if(window.count() >= 10):
start_date = window.min()
rdata = rdata[rdata['date']>=start_date]
t = rdata['date'].values
t = [float(date_day_diff(d, start_date)) for d in t]
y = rdata['deaths'].values
try:
bounds =([-1e6, -1e6, 0.001, 0.0], [1e6, 1e6, 0.999, t[-1]+16])
 popt, pcov = curve_fit(logistic, t, y, bounds = bounds)
 residuals = y - logistic(t, *popt)
 ss_res = np.sum(residuals**2)
 ss_tot = np.sum(( y - np.mean(y)) **2)
 rs = 1 -(ss_res / ss_tot)
 if plot:
print(st, co)
 print(popt)
 print('R squared: ', rs)
 T = np.arange(0, 60, 1 ).tolist()
yfit = logistic(T, *popt)
 pylab.plot(t, y, 'o')
 pylab.plot(T, yfit)
 pylab.show()
if rs>=0.95:
for index, rt in rtest.iterrows() :
tdate = rt['date']
prev_max = 0
if(tdate<=to_date):
ca = list(train[(train['date']==tdate)&(train['state']==st)&(train['country']==co)]['deaths'].values)[0]
prev_max = max(prev_max, ca)
 else:
ca = logistic([date_day_diff(tdate, start_date)], *popt)
 ca = max(prev_max, ca[0])
 prev_max = ca
predictions_deaths.append(( rt['id'], int(ca)))
 predicted = True
except:
pass
if not predicted:
t = rdata['date'].values
start_date = t[0]
t = np.array([float(date_day_diff(d, start_date)) for d in t])
 y = rdata['deaths'].values
linreg = LinearRegression()
linreg.fit(t[-10:].reshape(-1, 1), y[-10:])
 m = linreg.coef_[0]
b = linreg.intercept_
if plot:
print(st, co)
 print(linreg.intercept_, linreg.coef_)
 T = np.arange(0, 90, 1 ).tolist()
y_pred = [m*x+b for x in T]
pylab.plot(t, y, 'o')
 pylab.plot(T, y_pred)
 pylab.show()
for index, rt in rtest.iterrows() :
tdate = rt['date']
prev_max = 0
if(tdate<=to_date):
ca = list(train[(train['date']==tdate)&(train['state']==st)&(train['country']==co)]['deaths'].values)[0]
prev_max = max(prev_max, ca)
 else:
ca = m*date_day_diff(tdate, start_date)+ b
ca = max(ca, prev_max)
 prev_max = ca
predictions_deaths.append(( rt['id'], int(ca)) )
preds, model = runXGB(train_X, train_y, test_X, num_rounds=400)
 out_df = pd.DataFrame(preds)
 out_df.columns = ["high", "medium", "low"]
out_df["listing_id"] = test_df.listing_id.values
out_df.to_csv("xgb_starter2.csv", index=False )<install_modules>
data = {
'ForecastId': [pred[0] for pred in predictions_deaths],
'Fatalities': [pred[1] for pred in predictions_deaths],
}
df_deaths = pd.DataFrame(data, columns = data.keys())
 df_deaths.head()
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/<set_options>
df_submission = df_cases.join(df_deaths.set_index('ForecastId'), on = 'ForecastId')
 df_submission.to_csv('submission.csv', index=False)
 df_submission.head()