Kaggle competition: Click-Through Rate Prediction
tt['ConfirmedCases_Pred1'] = tt[['ConfirmedCases','ConfirmedCases_Pred1']].max(axis=1)
 tt['Fatalities_Pred1'] = tt[['Fatalities','Fatalities_Pred1']].max(axis=1)
 tt['ConfirmedCases_Pred1'] = tt['ConfirmedCases_Pred1'].fillna(0)
 tt['Fatalities_Pred1'] = tt['Fatalities_Pred1'].fillna(0)
 tt.loc[~tt['ConfirmedCases'].isna() , 'ConfirmedCases_Pred1'] = tt.loc[~tt['ConfirmedCases'].isna() ]['ConfirmedCases']
tt.loc[~tt['Fatalities'].isna() , 'Fatalities_Pred1'] = tt.loc[~tt['Fatalities'].isna() ]['Fatalities']
tt['ConfirmedCases_Pred1'] = tt.groupby('Place')['ConfirmedCases_Pred1'].transform('cummax')
 tt['Fatalities_Pred1'] = tt.groupby('Place')['Fatalities_Pred1'].transform('cummax' )<load_from_csv>
from catboost import CatBoostClassifier
ss = pd.read_csv('.. /input/covid19-global-forecasting-week-2/submission.csv' )<save_to_csv>
file_path = "/kaggle/input/avazu-ctr-prediction/train.gz"
data_df = pd.read_csv(file_path, header=0, nrows=10000000 ).sample(frac=1 )
mysub = tt.dropna(subset=['ForecastId'])[['ForecastId','ConfirmedCases_Pred1','Fatalities_Pred1']]
mysub['ForecastId'] = mysub['ForecastId'].astype('int')
 mysub = mysub.rename(columns={'ConfirmedCases_Pred1':'ConfirmedCases',
'Fatalities_Pred1': 'Fatalities'})
 mysub.to_csv('submission.csv', index=False )<load_from_csv>
train_count = int(len(data_df)* 0.9 )
path = '.. /input/covid19-global-forecasting-week-2/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'submission.csv')
 train['Date'] = train['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 test['Date'] = test['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 train['days'] =(train['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
test['days'] =(test['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
train.loc[train['Province_State'].isnull() , 'Province_State'] = 'N/A'
test.loc[test['Province_State'].isnull() , 'Province_State'] = 'N/A'
train['Area'] = train['Country_Region'] + '_' + train['Province_State']
test['Area'] = test['Country_Region'] + '_' + test['Province_State']
print(train['Date'].max())
 print(test['Date'].min())
 print(train['days'].max())
 N_AREAS = train['Area'].nunique()
AREAS = np.sort(train['Area'].unique())
 TRAIN_N = 70
print(train[train['days'] < TRAIN_N]['Date'].max())
 train.head()<compute_train_metric>
X_train = data_df.iloc[:train_count,2:]
y_train = data_df.iloc[:train_count,1]
def eval1(y, p):
val_len = y.shape[1] - TRAIN_N
return np.sqrt(mean_squared_error(y[:, TRAIN_N:TRAIN_N+val_len].flatten() , p[:, TRAIN_N:TRAIN_N+val_len].flatten()))
def run_c(params, X, test_size=50):
gr_base = []
gr_base_factor = []
x_min = np.ma.MaskedArray(X, X<1)
 x_min = x_min.argmin(axis=1)
 for i in range(X.shape[0]):
temp = X[i,:]
threshold = np.log(1+params['min cases for growth rate'])
 num_days = params['last N days']
if(temp > threshold ).sum() > num_days:
gr_base.append(np.clip(np.diff(temp[temp > threshold])[-num_days:].mean() , 0, params['growth rate max']))
gr_base_factor.append(np.clip(np.diff(np.diff(temp[temp > threshold])) [-num_days:].mean() , -0.2, params["growth rate factor max"]))
else:
gr_base.append(params['growth rate default'])
 gr_base_factor.append(params['growth rate factor'])
 gr_base = np.array(gr_base)
 gr_base_factor = np.array(gr_base_factor)
 preds = X.copy()
for i in range(test_size):
delta = np.clip(preds[:, -1], np.log(2), None)+ gr_base *(1 + params['growth rate factor']*(1 + params['growth rate factor factor'])**(i)) **(i)
 preds = np.hstack(( preds, delta.reshape(-1,1)))
 return preds
params = {
"min cases for growth rate": 30,
"last N days": 5,
"growth rate default": 0.25,
"growth rate max": 0.3,
"growth rate factor max": -0.01,
"growth rate factor": -0.05,
"growth rate factor factor": 0.005,
}
x = train_p_c[train_p_c.index=="China_Qinghai"]
x = train_p_c
preds_c_1 = run_c(params, np.log(1+x.values)[:,:TRAIN_N])
<compute_test_metric>
X_val = data_df.iloc[train_count:,2:]
y_val = data_df.iloc[train_count:,1]
warnings.filterwarnings("ignore")
 def f(x, K, a, x0):
return K * x ** a * np.exp(-x/x0)
 def eval1(y, p):
val_len = y.shape[1] - TRAIN_N
return np.sqrt(mean_squared_error(y[:, TRAIN_N:TRAIN_N+val_len].flatten() , p[:, TRAIN_N:TRAIN_N+val_len].flatten()))
def run(params, X, X_change, test_size=50):
print(X)
 x_mins = np.ma.MaskedArray(X, X<10)
 x_mins = x_mins.argmin(axis=1)
 print(x_mins)
 popts = []
scores = []
for i in tqdm(range(X_change.shape[0])) :
if X[i,:].sum() == 0:
x_mins[i] = X.shape[1] - 1
best_score = 100
best_popt = [1,1,1]
x_min_max = X_change.shape[1]
x_min_max = np.minimum(x_mins[i] + 10, X_change.shape[1])
 early_stopping = 5
early_stopping_count = 0
for x_min in np.arange(x_mins[i], x_min_max):
x = np.arange(x_min, X_change.shape[1])- x_min
y = X_change[i, x_min:]
try:
popt, pcov = curve_fit(f, x,y, bounds=(0, [10, 5, np.max(x)]))
except:
popt = [1,1,1]
p = np.zeros(X_change.shape[1])
 p[x_min:] = f(x, *popt)
 p = np.cumsum(p, axis=0)
 score = np.sqrt(mean_squared_error(np.log1p(X[i,:]), np.log1p(p)))
 if score < best_score:
best_score = score
best_popt = popt
x_mins[i] = x_min
else:
early_stopping_count += 1
if early_stopping_count >= early_stopping:
continue
popts.append(best_popt)
 scores.append(best_score)
 preds = X_change.copy()
preds_new = np.zeros(( X.shape[0], test_size))
for i in range(X.shape[0]):
x = np.arange(X.shape[1], test_size+X.shape[1])- x_mins[i]
y = f(x, *popts[i])
 y = y[-test_size-X.shape[1]:]
preds_new[i,:] = y
preds = np.hstack(( preds, preds_new))
preds = np.cumsum(preds, axis=1)
 return preds, x_mins, scores
x1 = train_p_c
x = train_p_c.values[:,:TRAIN_N]
x_c = train_p_c_change.values[:,:TRAIN_N]
params = {}
preds_c_pl, x_mins, scores = run(params, x, x_c)
 preds_c_pl = np.log1p(preds_c_pl)
<prepare_output>
cat_features = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
preds_c = preds_c_1.copy()
idx = np.where(x_mins<100)
 preds_c[idx] = 0.8 * preds_c_1[idx] + 0.2 * preds_c_pl[idx]
for i in range(N_AREAS):
if 'China' in AREAS[i] and preds_c[i, TRAIN_N-1] < np.log(31):
preds_c[i, TRAIN_N:] = preds_c[i, TRAIN_N-1] 
<prepare_x_and_y>
model = CatBoostClassifier(
 iterations=50,
learning_rate=0.5,
task_type='GPU',
loss_function='Logloss',
)
f_rate =(train_p_f / train_p_c ).fillna(0)
 X_c = np.log(1+train_p_c.values)[:,:TRAIN_N]
X_f = train_p_f.values[:,:TRAIN_N]<normalization>
model.fit(
 X_train, y_train,
eval_set=(X_val, y_val),
cat_features=cat_features,
verbose=10,
)
def lin_w(sz):
res = np.linspace(0, 1, sz+1, endpoint=False)[1:]
return np.append(res, np.append([1], res[::-1]))
def run_f(params, X_c, X_f, X_f_r, test_size=50):
X_f_r = np.array(np.ma.mean(np.ma.masked_outside(X_f_r, 0.06, 0.4)[:,:], axis=1))
X_f_r = np.clip(X_f_r, params['fatality_rate_lower'], params['fatality_rate_upper'])
 X_c = np.clip(np.exp(X_c)-1, 0, None)
 preds = X_f.copy()
train_size = X_f.shape[1] - 1
for i in range(test_size):
t_lag = train_size+i-params['length']
t_wsize = 3
delta = np.average(np.diff(X_c, axis=1)[:, t_lag-t_wsize:t_lag+1+t_wsize], axis=1)
 delta = params['absolute growth'] + delta * X_f_r
preds = np.hstack(( preds, preds[:, -1].reshape(-1,1)+ delta.reshape(-1,1)))
 return preds
params = {
"length": 6,
"absolute growth": 0.02,
"fatality_rate_lower": 0.035,
"fatality_rate_upper": 0.40,
}
preds_f_1 = run_f(params, preds_c, X_f, f_rate.values[:,:TRAIN_N])
 preds_f_1 = np.log(1+preds_f_1)
 preds_f_1 = np.log1p(0.9*(np.exp(preds_f_1)-1)) 
<prepare_output>
model.get_feature_importance(prettified=True )
preds_f = preds_f_1<compute_test_metric>
test_file_path = "/kaggle/input/avazu-ctr-prediction/test.gz"
test_df = pd.read_csv(test_file_path, header=0, dtype=str )
if False:
val_len = train_p_c.values.shape[1] - TRAIN_N
for i in range(val_len):
d = i + TRAIN_N
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c.values[:, d]), preds_c[:, d]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f.values[:, d]), preds_f[:, d]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_c[:, TRAIN_N:TRAIN_N+val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_f[:, TRAIN_N:TRAIN_N+val_len].flatten()))
print(f"{(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]" )<categorify>
y_test = model.predict(X_test,
prediction_type='Probability',
ntree_start=0, ntree_end=model.get_best_iteration() ,
thread_count=-1, verbose=None )
temp = pd.DataFrame(np.clip(np.exp(preds_c)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="ConfirmedCases")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 temp = pd.DataFrame(np.clip(np.exp(preds_f)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="Fatalities")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 test.head()<save_to_csv>
id_test.join(pd.DataFrame(y_test))
test.to_csv("submission.csv", index=False, columns=["ForecastId", "ConfirmedCases", "Fatalities"] )<sort_values>
submission_df = pd.read_csv("/kaggle/input/avazu-ctr-prediction/sampleSubmission.gz" )
<set_options><EOS> 
submission_df["click"] = y_test[:, 1]
submission_df.to_csv("submission.csv", index=False)
 submission_df.head()