Kaggle competition: Coupon Purchase Prediction
<SOS> metric: MAP@{K} Kaggle data source: coupon-purchase-prediction<data_type_conversions>
!pip install turicreate
X_Train = df_train.copy()
X_Train['State'].fillna(EMPTY_VAL, inplace=True)
 X_Train['State'] = X_Train.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Train.loc[:, 'Date'] = X_Train.Date.dt.strftime("%m%d")
 X_Train["Date"] = X_Train["Date"].astype(int)
 X_Train.head()<data_type_conversions>
import os
import pandas as pd
import numpy as np
from zipfile import ZipFile
import turicreate
X_Test = df_test.copy()
X_Test['State'].fillna(EMPTY_VAL, inplace=True)
 X_Test['State'] = X_Test.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Test.loc[:, 'Date'] = X_Test.Date.dt.strftime("%m%d")
 X_Test["Date"] = X_Test["Date"].astype(int)
 X_Test.head()<categorify>
ds_dir = '.. /input/coupon-purchase-prediction'
le = preprocessing.LabelEncoder()
X_Train.Country = le.fit_transform(X_Train.Country)
 X_Train['State'] = le.fit_transform(X_Train['State'])
 X_Test.Country = le.fit_transform(X_Test.Country)
 X_Test['State'] = le.fit_transform(X_Test['State'] )<filter>
with ZipFile(os.path.join(ds_dir,"coupon_detail_train.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"coupon_list_test.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"coupon_list_train.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"coupon_visit_train.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"sample_submission.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"user_list.csv.zip"), 'r')as zipObj:
zipObj.extractall()
df_train.loc[df_train.Country == 'Afghanistan', :]<set_options>
cd_train = pd.read_csv('coupon_detail_train.csv')
 cl_test = pd.read_csv('coupon_list_test.csv')
 cl_train = pd.read_csv('coupon_list_train.csv')
 sample_sub = pd.read_csv('sample_submission.csv')
 user_list = pd.read_csv('user_list.csv' )
filterwarnings('ignore' )<prepare_x_and_y>
cd_train = cd_train.groupby(['USER_ID_hash', 'COUPON_ID_hash'])['PURCHASEID_hash'].count().to_frame().reset_index()
countries = X_Train.Country.unique()
df_out = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_Train.loc[X_Train.Country == country, :].State.unique()
for state in states:
X_Train_CS = X_Train.loc[(X_Train.Country == country)&(X_Train.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y1_Train_CS = X_Train_CS.loc[:, 'ConfirmedCases']
y2_Train_CS = X_Train_CS.loc[:, 'Fatalities']
X_Train_CS = X_Train_CS.loc[:, ['State', 'Country', 'Date']]
X_Train_CS.Country = le.fit_transform(X_Train_CS.Country)
 X_Train_CS['State'] = le.fit_transform(X_Train_CS['State'])
 X_Test_CS = X_Test.loc[(X_Test.Country == country)&(X_Test.State == state), ['State', 'Country', 'Date', 'ForecastId']]
X_Test_CS_Id = X_Test_CS.loc[:, 'ForecastId']
X_Test_CS = X_Test_CS.loc[:, ['State', 'Country', 'Date']]
X_Test_CS.Country = le.fit_transform(X_Test_CS.Country)
 X_Test_CS['State'] = le.fit_transform(X_Test_CS['State'])
 model1 = XGBRegressor(learning_rate=0.05, n_estimators=2000,cv=10)
 model1.fit(X_Train_CS, y1_Train_CS)
 y1_pred = model1.predict(X_Test_CS)
 model2 = XGBRegressor(learning_rate=0.05, n_estimators=2000,cv=10)
 model2.fit(X_Train_CS, y2_Train_CS)
 y2_pred = model2.predict(X_Test_CS)
 df = pd.DataFrame({'ForecastId': X_Test_CS_Id, 'ConfirmedCases': y1_pred, 'Fatalities': y2_pred})
 df_out = pd.concat([df_out, df], axis=0 )<data_type_conversions>
cl_train.drop(['VALIDFROM','VALIDEND'],axis=1,inplace=True)
 cl_test.drop(['VALIDFROM','VALIDEND'],axis=1,inplace=True)
 cl_train.VALIDPERIOD.fillna(180, inplace=True)
 cl_test.VALIDPERIOD.fillna(180, inplace=True)
 cl_train.fillna(1, inplace=True)
 cl_test.fillna(1, inplace=True )
df_out.ForecastId = df_out.ForecastId.astype('int')
 df_out.tail()<save_to_csv>
observation_data = turicreate.SFrame(cd_train)
 item_data = turicreate.SFrame(pd.concat([cl_train,cl_test]))
df_out.to_csv('submission.csv', index=False )<import_modules>
users = turicreate.SFrame(user_list[['USER_ID_hash']])
 items = turicreate.SFrame(cl_test[['COUPON_ID_hash']] )
import numpy as np
import pandas as pd
import lightgbm as lgb
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit<load_from_csv>
cosine_model = turicreate.recommender.item_content_recommender.create(similarity_metrics='cosine', item_data=item_data, item_id='COUPON_ID_hash', observation_data=observation_data, target='PURCHASEID_hash', user_id='USER_ID_hash', item_data_transform='auto', verbose=True)
 jaccard_model = turicreate.recommender.item_content_recommender.create(similarity_metrics='jaccard', item_data=item_data, item_id='COUPON_ID_hash', observation_data=observation_data, target='PURCHASEID_hash', user_id='USER_ID_hash', item_data_transform='auto', verbose=True)
 pearson_model = turicreate.recommender.item_content_recommender.create(similarity_metrics='pearson', item_data=item_data, item_id='COUPON_ID_hash', observation_data=observation_data, target='PURCHASEID_hash', user_id='USER_ID_hash', item_data_transform='auto', verbose=True )
train = pd.read_csv('.. /input/covid19-global-forecasting-week-2/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-2/test.csv')
 world_population = pd.read_csv('.. /input/population-by-country-2020/population_by_country_2020.csv' )<merge>
cosine_res = cosine_model.recommend(users, k=10, items=items ).to_dataframe()
jaccard_res = jaccard_model.recommend(users, k=10, items=items ).to_dataframe()
pearson_res = pearson_model.recommend(users, k=10, items=items ).to_dataframe()
world_population = world_population[['Country(or dependency)', 'Population(2020)']]
world_population.columns = ['Country(or dependency)', 'Population']
world_population.loc[world_population['Country(or dependency)']=='United States', 'Country(or dependency)'] = 'US'
train = train.merge(world_population, left_on='Country_Region', right_on='Country(or dependency)', how='left')
 test = test.merge(world_population, left_on='Country_Region', right_on='Country(or dependency)', how='left' )<feature_engineering>
def clean_prediction(row):
data = row.PURCHASED_COUPONS
data = str("".join(str(data)) [2:-2].replace("', '"," "))
return data
train['State_Country'] = [s + '_' + c if s == s else c for s,c in train[['Province_State', 'Country_Region']].values ]
test['State_Country'] = [s + '_' + c if s == s else c for s,c in test[['Province_State', 'Country_Region']].values ]<feature_engineering>
cosine_res = cosine_res.groupby('USER_ID_hash')['COUPON_ID_hash'].apply(list ).reset_index(name='PURCHASED_COUPONS')
 jaccard_res = jaccard_res.groupby('USER_ID_hash')['COUPON_ID_hash'].apply(list ).reset_index(name='PURCHASED_COUPONS')
 pearson_res = pearson_res.groupby('USER_ID_hash')['COUPON_ID_hash'].apply(list ).reset_index(name='PURCHASED_COUPONS' )
train.loc[(train['Date']=='2020-03-24')&(train['State_Country']=='France'),'ConfirmedCases'] = 22654
train.loc[(train['Date']=='2020-03-24')&(train['State_Country']=='France'),'Fatalities'] = 1000<groupby>
cosine_res['PURCHASED_COUPONS'] = cosine_res.apply(clean_prediction, axis=1)
 jaccard_res['PURCHASED_COUPONS'] = jaccard_res.apply(clean_prediction, axis=1)
 pearson_res['PURCHASED_COUPONS'] = pearson_res.apply(clean_prediction, axis=1 )
for metric in ['ConfirmedCases', 'Fatalities']:
dict_values = train.groupby('State_Country')[metric].apply(np.array ).to_dict()
for country in dict_values:
if sum(np.diff(dict_values[country])< 0):
print(country, metric)
 new_val = [dict_values[country][-1]]
for val_1, val_2 in zip(dict_values[country][1:][::-1], dict_values[country][:-1][::-1]):
if val_2 <= new_val[-1]:
new_val += [val_2]
else:
new_val += [new_val[-1]]
new_val = np.array(new_val[::-1])
 train.loc[train.State_Country == country, metric] = new_val<feature_engineering>
cosine_res.to_csv('sub_cpp_turi_item_content_cosine.csv', index=False)
 jaccard_res.to_csv('sub_cpp_turi_item_content_jaccard.csv', index=False)
 pearson_res.to_csv('sub_cpp_turi_item_content_pearson.csv', index=False )
train['max_case'] = train['State_Country'].map(train.groupby('State_Country' ).ConfirmedCases.max())
 train['pct_c'] = train.ConfirmedCases.pct_change()<predict_on_test>
!pip install turicreate
def predict(data, country, len_predict, metrics, len_intersection):
country_data = data[data['State_Country']==country]
if country_data[metrics].max() <= 50:
start = 0
else:
start = 1
country_data = country_data.iloc[dict_case_date[country][start]:, :]
x_data = range(len(country_data.index))
y_data = country_data[metrics].values
if len(x_data)<= 1:
y_min = 0
y_max = 100
y_val = np.arange(len(x_data), len(x_data)+ len_predict)
 k = 100 if metrics == 'ConfirmedCases' else 2
return [-1, -1, -1], log_curve(y_val, 0.3, 30, 2)
 else:
add_min =(1 + country_data.pct_c.values[-3:].mean())**(1 / 2)
 add_max =(1 + country_data.pct_c.values[-3:].max())**(1 / 2)
 day_left = max(10, 70 - len(x_data))
y_min = y_data[-1] *(add_min ** day_left)
 k = 0.05 if metrics == 'ConfirmedCases' else 0.05 * 0.01
if y_min > country_data['Population'].values[0] * k:
y_min = country_data['Population'].values[0] * k
y_max = y_data[-1] *(add_max ** day_left + 30)
 k = 0.15 if metrics == 'ConfirmedCases' else 0.15 * 0.01
if y_max > country_data['Population'].values[0] * k:
y_max = country_data['Population'].values[0] * k
if add_min == add_max:
y_min = y_data[-1]
y_max = y_data[-1] *(1.3 ** 30)
 if y_data.max() < 100 and metrics == 'ConfirmedCases':
y_min = 0
y_max = 50000
if y_data.max() < 5 and metrics == 'Fatalities':
y_min = 0
y_max = 100
if y_max > 250000 and metrics == 'ConfirmedCases':
y_min = 1000
y_max = 250000
if y_max > 20000 and metrics == 'Fatalities':
y_min = 100
y_max = 20000
popt, pcov = curve_fit(log_curve, x_data, y_data, bounds=([0.1, 10, y_min ],[0.35, 50, y_max]),
p0=[0.2,30,(y_min + y_max)/ 2], maxfev=10000)
 y_val = np.arange(len(x_data)- len_intersection, len(x_data)+ len_predict - len_intersection)
 return popt, log_curve(y_val, popt[0], popt[1], popt[2])
 def log_curve(x, k, x_0, ymax):
return ymax /(1 + np.exp(-k*(x-x_0)) )<groupby>
import os
import pandas as pd
import numpy as np
from zipfile import ZipFile
import turicreate
metrics = 'ConfirmedCases'
dict_values = train.groupby('State_Country')[metrics].apply(np.array ).to_dict()
dict_case_date = {}
for country in dict_values:
dict_case_date[country] = []
for case in [1, 10, 100, 250, 500, 1000, 2500, 5000]:
try:
dict_case_date[country] += [np.where(dict_values[country] >= case)[0][0]]
except:
dict_case_date[country] += [-1]
dict_case_date[country] = np.array(dict_case_date[country])
 dict_predict = {}
data_train = train.copy()
data_val = test.copy()
len_predict = data_val[data_val.State_Country == country].shape[0]
len_intersection = len(set(data_train.Date.unique())& set(data_val.Date.unique()))
for country in train.State_Country.unique() :
if data_train[data_train.State_Country == country][metrics].max() <= 10:
popt, pred_values = predict(data_train, country, len_predict, metrics, len_intersection)
 else:
popt, pred_values = predict(data_train, country, len_predict, metrics, len_intersection)
 dict_predict[country] = pred_values<feature_engineering>
ds_dir = '.. /input/coupon-purchase-prediction'
test['ConfirmedCases'] = 0
for country in test['State_Country'].unique() :
test.loc[test.State_Country == country, 'ConfirmedCases'] = dict_predict[country]<groupby>
with ZipFile(os.path.join(ds_dir,"coupon_detail_train.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"coupon_list_test.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"coupon_list_train.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"coupon_visit_train.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"sample_submission.csv.zip"), 'r')as zipObj:
zipObj.extractall()
with ZipFile(os.path.join(ds_dir,"user_list.csv.zip"), 'r')as zipObj:
zipObj.extractall()
metrics = 'Fatalities'
dict_values = train.groupby('State_Country')[metrics].apply(np.array ).to_dict()
dict_case_date = {}
for country in dict_values:
dict_case_date[country] = []
for case in [1, 10, 100, 250, 500, 1000, 2500, 5000]:
try:
dict_case_date[country] += [np.where(dict_values[country] >= case)[0][0]]
except:
dict_case_date[country] += [-1]
dict_case_date[country] = np.array(dict_case_date[country])
 dict_predict = {}
data_train = train.copy()
data_val = test.copy()
len_predict = data_val[data_val.State_Country == country].shape[0]
len_intersection = len(set(data_train.Date.unique())& set(data_val.Date.unique()))
for country in train.State_Country.unique() :
if data_train[data_train.State_Country == country][metrics].max() <= 3:
popt, pred_values = predict(data_train, country, len_predict, metrics, len_intersection)
 else:
popt, pred_values = predict(data_train, country, len_predict, metrics, len_intersection)
 dict_predict[country] = pred_values<feature_engineering>
cd_train = pd.read_csv('coupon_detail_train.csv')
 cl_test = pd.read_csv('coupon_list_test.csv')
 cl_train = pd.read_csv('coupon_list_train.csv')
 sample_sub = pd.read_csv('sample_submission.csv')
 user_list = pd.read_csv('user_list.csv' )
test['Fatalities'] = 0
for country in test['State_Country'].unique() :
test.loc[test.State_Country == country, 'Fatalities'] = dict_predict[country]<save_to_csv>
cd_train = cd_train.groupby(['USER_ID_hash', 'COUPON_ID_hash'])['PURCHASEID_hash'].count().to_frame().reset_index()
submit = pd.read_csv('.. /input/covid19-global-forecasting-week-2/submission.csv')
 submit['Fatalities'] = test['Fatalities'].astype('float')
 submit['ConfirmedCases'] = test['ConfirmedCases'].astype('float')
 submit.to_csv('submission.csv',index=False )<compute_test_metric>
cl_train.drop(['VALIDFROM','VALIDEND'],axis=1,inplace=True)
 cl_test.drop(['VALIDFROM','VALIDEND'],axis=1,inplace=True)
 cl_train.VALIDPERIOD.fillna(180, inplace=True)
 cl_test.VALIDPERIOD.fillna(180, inplace=True)
 cl_train.fillna(1, inplace=True)
 cl_test.fillna(1, inplace=True )
def rmsle(true, pred):
true = np.array(true)
 pred = np.array(pred)
 return np.mean(( np.log1p(true)- np.log1p(pred)) ** 2)**(1/2 )<merge>
observation_data = turicreate.SFrame(cd_train)
 item_data = turicreate.SFrame(pd.concat([cl_train,cl_test]))
new_test = test.merge(train[['State_Country', 'Date', 'Fatalities']], on = ['State_Country', 'Date'])
 rmsle(new_test['Fatalities_y'], new_test['Fatalities_x'] )<merge>
users = turicreate.SFrame(user_list[['USER_ID_hash']])
 items = turicreate.SFrame(cl_test[['COUPON_ID_hash']] )
new_test = test.merge(train[['State_Country', 'Date', 'ConfirmedCases']], on = ['State_Country', 'Date'])
 rmsle(new_test['ConfirmedCases_y'], new_test['ConfirmedCases_x'] )<import_modules>
cosine_model = turicreate.recommender.item_content_recommender.create(similarity_metrics='cosine', item_data=item_data, item_id='COUPON_ID_hash', observation_data=observation_data, target='PURCHASEID_hash', user_id='USER_ID_hash', item_data_transform='auto', verbose=True)
 jaccard_model = turicreate.recommender.item_content_recommender.create(similarity_metrics='jaccard', item_data=item_data, item_id='COUPON_ID_hash', observation_data=observation_data, target='PURCHASEID_hash', user_id='USER_ID_hash', item_data_transform='auto', verbose=True)
 pearson_model = turicreate.recommender.item_content_recommender.create(similarity_metrics='pearson', item_data=item_data, item_id='COUPON_ID_hash', observation_data=observation_data, target='PURCHASEID_hash', user_id='USER_ID_hash', item_data_transform='auto', verbose=True )
import xgboost as xgb
from datetime import datetime
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import LinearRegression, BayesianRidge<load_from_csv>
cosine_res = cosine_model.recommend(users, k=10, items=items ).to_dataframe()
jaccard_res = jaccard_model.recommend(users, k=10, items=items ).to_dataframe()
pearson_res = pearson_model.recommend(users, k=10, items=items ).to_dataframe()
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv')
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv' )<count_missing_values>
def clean_prediction(row):
data = row.PURCHASED_COUPONS
data = str("".join(str(data)) [2:-2].replace("', '"," "))
return data
total = train.isnull().sum()
print(total )<count_missing_values>
cosine_res = cosine_res.groupby('USER_ID_hash')['COUPON_ID_hash'].apply(list ).reset_index(name='PURCHASED_COUPONS')
 jaccard_res = jaccard_res.groupby('USER_ID_hash')['COUPON_ID_hash'].apply(list ).reset_index(name='PURCHASED_COUPONS')
 pearson_res = pearson_res.groupby('USER_ID_hash')['COUPON_ID_hash'].apply(list ).reset_index(name='PURCHASED_COUPONS' )
k = train[train['Province_State'].isna() ]
print(k )<count_missing_values>
cosine_res['PURCHASED_COUPONS'] = cosine_res.apply(clean_prediction, axis=1)
 jaccard_res['PURCHASED_COUPONS'] = jaccard_res.apply(clean_prediction, axis=1)
 pearson_res['PURCHASED_COUPONS'] = pearson_res.apply(clean_prediction, axis=1 )
<feature_engineering><EOS> 
cosine_res.to_csv('sub_cpp_turi_item_content_cosine.csv', index=False)
 jaccard_res.to_csv('sub_cpp_turi_item_content_jaccard.csv', index=False)
 pearson_res.to_csv('sub_cpp_turi_item_content_pearson.csv', index=False )