Kaggle competition: COVID19 Global Forecasting (Week 4)
dupes = items[(items.duplicated(subset=['item_name','category_id'],keep=False)) ]
dupes['in_test'] = dupes.item_id.isin(test.item_id.unique())
 dupes = dupes.groupby('item_name' ).agg({'item_id':['first','last'],'in_test':['first','last']})
 dupes = dupes[(dupes[('in_test', 'first')]==False)|(dupes[('in_test', 'last')]==False)]
temp = dupes[dupes[('in_test', 'first')]==True]
keep_first = dict(zip(temp[('item_id', 'last')], temp[('item_id', 'first')]))
temp = dupes[dupes[('in_test', 'first')]==False]
keep_second = dict(zip(temp[('item_id', 'first')], temp[('item_id', 'last')]))
item_map = {**keep_first, **keep_second}<feature_engineering>
from pandas_profiling import ProfileReport
sales = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 sales =(sales 
.query('0 < item_price < 50000 and 0 < item_cnt_day < 1001')
.replace({
'shop_id':{0:57, 1:58, 11:10},
'item_id':item_map
})
)
 sales = sales[sales['shop_id'].isin(test.shop_id.unique())]
sales['date'] = pd.to_datetime(sales.date,format='%d.%m.%Y')
 sales['weekday'] = sales.date.dt.dayofweek
sales['first_sale_day'] = sales.date.dt.dayofyear
sales['first_sale_day'] += 365 *(sales.date.dt.year-2013)
 sales['first_sale_day'] = sales.groupby('item_id')['first_sale_day'].transform('min' ).astype('int16')
 sales['revenue'] = sales['item_cnt_day']*sales['item_price']<feature_engineering>
xtrain = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 xtest = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 xsubmission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )
temp = sales.groupby(['shop_id','weekday'] ).agg({'item_cnt_day':'sum'} ).reset_index()
temp = pd.merge(temp, sales.groupby(['shop_id'] ).agg({'item_cnt_day':'sum'} ).reset_index() , on='shop_id', how='left')
 temp.columns = ['shop_id','weekday', 'shop_day_sales', 'shop_total_sales']
temp['day_quality'] = temp['shop_day_sales']/temp['shop_total_sales']
temp = temp[['shop_id','weekday','day_quality']]
dates = pd.DataFrame(data={'date':pd.date_range(start='2013-01-01',end='2015-11-30')})
 dates['weekday'] = dates.date.dt.dayofweek
dates['month'] = dates.date.dt.month
dates['year'] = dates.date.dt.year - 2013
dates['date_block_num'] = dates['year']*12 + dates['month'] - 1
dates['first_day_of_month'] = dates.date.dt.dayofyear
dates['first_day_of_month'] += 365 * dates['year']
dates = dates.join(temp.set_index('weekday'), on='weekday')
 dates = dates.groupby(['date_block_num','shop_id','month','year'] ).agg({'day_quality':'sum','first_day_of_month':'min'} ).reset_index()
dates.query('shop_id == 28' ).head(15 )<groupby>
train_profile = ProfileReport(xtrain, title='Pandas Profiling Report', html={'style':{'full_width':True}})
 train_profile
sales =(sales 
.groupby(['date_block_num', 'shop_id', 'item_id'])
.agg({
'item_cnt_day':'sum',
'revenue':'sum',
'first_sale_day':'first'
})
.reset_index() 
.rename(columns={'item_cnt_day':'item_cnt'})
)
 sales.sample(5 )<drop_column>
xtrain.rename(columns={'Country_Region':'Country'}, inplace=True)
 xtest.rename(columns={'Country_Region':'Country'}, inplace=True)
 xtrain.rename(columns={'Province_State':'State'}, inplace=True)
 xtest.rename(columns={'Province_State':'State'}, inplace=True)
 xtrain['Date'] = pd.to_datetime(xtrain['Date'], infer_datetime_format=True)
 xtest['Date'] = pd.to_datetime(xtest['Date'], infer_datetime_format=True)
 xtrain.info()
xtest.info()
y1_xTrain = xtrain.iloc[:, -2]
y1_xTrain.head()
y2_xTrain = xtrain.iloc[:, -1]
y2_xTrain.head()
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state
test['date_block_num'] = 34
del test['ID']<concatenate>
X_xTrain = xtrain.copy()
X_xTrain['State'].fillna(EMPTY_VAL, inplace=True)
 X_xTrain['State'] = X_xTrain.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_xTrain.loc[:, 'Date'] = X_xTrain.Date.dt.strftime("%m%d")
 X_xTrain["Date"] = X_xTrain["Date"].astype(int)
 X_xTrain.head()
X_xTest = xtest.copy()
X_xTest['State'].fillna(EMPTY_VAL, inplace=True)
 X_xTest['State'] = X_xTest.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_xTest.loc[:, 'Date'] = X_xTest.Date.dt.strftime("%m%d")
 X_xTest["Date"] = X_xTest["Date"].astype(int)
 X_xTest.head()
df = pd.concat([df,test] ).fillna(0)
 df = df.reset_index()
del df['index']<merge>
le = preprocessing.LabelEncoder()
X_xTrain.Country = le.fit_transform(X_xTrain.Country)
 X_xTrain['State'] = le.fit_transform(X_xTrain['State'])
 X_xTrain.head()
X_xTest.Country = le.fit_transform(X_xTest.Country)
 X_xTest['State'] = le.fit_transform(X_xTest['State'])
 X_xTest.head()
xtrain.head()
xtrain.loc[xtrain.Country == 'Afghanistan', :]
xtest.tail()
df = pd.merge(df, sales, on=['shop_id', 'item_id', 'date_block_num'], how='left' ).fillna(0)
 df = pd.merge(df, dates, on=['date_block_num','shop_id'], how='left')
 df = pd.merge(df, items.drop(columns=['item_name','group_name','category_name']), on='item_id', how='left' )<feature_engineering>
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
countries = X_xTrain.Country.unique()
shops = pd.read_csv('.. /input/predict-future-sales-eng-translation/shops.csv')
 shops_cats = pd.DataFrame(
 np.array(list(product(*[df['shop_id'].unique() , df['category_id'].unique() ]))),
columns =['shop_id', 'category_id'] 
)
 temp = df.groupby(['category_id', 'shop_id'] ).agg({'item_cnt':'sum'} ).reset_index()
temp2 = temp.groupby('shop_id' ).agg({'item_cnt':'sum'} ).rename(columns={'item_cnt':'shop_total'})
 temp = temp.join(temp2, on='shop_id')
 temp['category_proportion'] = temp['item_cnt']/temp['shop_total']
temp = temp[['shop_id', 'category_id', 'category_proportion']]
shops_cats = pd.merge(shops_cats, temp, on=['shop_id','category_id'], how='left')
 shops_cats = shops_cats.fillna(0)
 shops_cats = shops_cats.pivot(index='shop_id', columns=['category_id'])
 kmeans = KMeans(n_clusters=7, random_state=0 ).fit(shops_cats)
 shops_cats['shop_cluster'] = kmeans.labels_.astype('int8')
 shops = shops.join(shops_cats['shop_cluster'], on='shop_id' )<feature_engineering>
xout = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_xTrain.loc[X_xTrain.Country == country, :].State.unique()
for state in states:
X_xTrain_CS = X_xTrain.loc[(X_xTrain.Country == country)&(X_xTrain.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y1_xTrain_CS = X_xTrain_CS.loc[:, 'ConfirmedCases']
y2_xTrain_CS = X_xTrain_CS.loc[:, 'Fatalities']
X_xTrain_CS = X_xTrain_CS.loc[:, ['State', 'Country', 'Date']]
X_xTrain_CS.Country = le.fit_transform(X_xTrain_CS.Country)
 X_xTrain_CS['State'] = le.fit_transform(X_xTrain_CS['State'])
 X_xTest_CS = X_xTest.loc[(X_xTest.Country == country)&(X_xTest.State == state), ['State', 'Country', 'Date', 'ForecastId']]
X_xTest_CS_Id = X_xTest_CS.loc[:, 'ForecastId']
X_xTest_CS = X_xTest_CS.loc[:, ['State', 'Country', 'Date']]
X_xTest_CS.Country = le.fit_transform(X_xTest_CS.Country)
 X_xTest_CS['State'] = le.fit_transform(X_xTest_CS['State'])
 xmodel1 = XGBRegressor(n_estimators=1000)
 xmodel1.fit(X_xTrain_CS, y1_xTrain_CS)
 y1_xpred = xmodel1.predict(X_xTest_CS)
 xmodel2 = XGBRegressor(n_estimators=1000)
 xmodel2.fit(X_xTrain_CS, y2_xTrain_CS)
 y2_xpred = xmodel2.predict(X_xTest_CS)
 xdata = pd.DataFrame({'ForecastId': X_xTest_CS_Id, 'ConfirmedCases': y1_xpred, 'Fatalities': y2_xpred})
 xout = pd.concat([xout, xdata], axis=0)
<merge><EOS> 
xout.ForecastId = xout.ForecastId.astype('int')
 xout.tail()
xout.to_csv('submission.csv', index=False )