Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<predict_on_test>
warnings.filterwarnings("ignore")
preds = mean_predictions(models, X_test_moa )<define_variables>
datapath = '.. /input/covid19-global-forecasting-week-2/'
train = pd.read_csv(datapath+'train.csv',)
 test = pd.read_csv(datapath+'test.csv' )
indexes = dict(zip(X_test_moa.index, range(X_test_moa.shape[0])) )<feature_engineering>
train['Date'] = train['Date'].astype('datetime64[ns]')
 test['Date'] = test['Date'].astype('datetime64[ns]')
 print("Train Date type: ", train['Date'].dtype)
 print("Test Date type: ",test['Date'].dtype )
for idx, row in submission.iterrows() :
sig_id = row.sig_id
if sig_id in indexes.keys() :
submission.loc[idx] = [sig_id] + [np.maximum(np.minimum(pred, 1 - 1e-3), 1e-3)for pred in preds[indexes[sig_id]]]
else:
submission.loc[idx] = [sig_id] + [0] * preds.shape[1]<save_to_csv>
train.columns = ['id','state','country','date','ConfirmedCases','Fatalities']
test.columns = ['ForecastId', 'state','country','date']
submission.to_csv('submission.csv', index=False )<save_to_csv>
train['place'] = train['state'].fillna('')+ '_' + train['country']
test['place'] = test['state'].fillna('')+ '_' + test['country']
submission.to_csv('submission.csv', index=False )<import_modules>
print('How many places?: ', 'Train: ', len(train['place'].unique()),
'Test: ', len(test['place'].unique()))
print('Unique place similar as test?: ',(train['place'].unique() == test['place'].unique() ).sum() )
import numpy as np
import pandas as pd<load_from_csv>
china_cases = train[train['place'].str.contains('China')][['date',
'ConfirmedCases',
'Fatalities']].reset_index(drop=True)
 restworld_cases = train[-train['place'].str.contains('China')][['date',
'ConfirmedCases',
'Fatalities']].reset_index(drop=True )
df1 = pd.read_csv('.. /input/moa-fn-pzad/submission.csv')
 df2 = pd.read_csv('.. /input/mechanisms-of-action-moa-tutorial/submission.csv' )<sort_values>
us_cases = train[train['place'].str.contains('US')][['date','place',
'ConfirmedCases',
'Fatalities']].reset_index(drop=True )
def order_sub(sub):
return sub.sort_values('sig_id' ).reset_index(drop=True )<sort_values>
def RMSLE(predicted, actual):
return np.sqrt(np.mean(np.power(( np.log(predicted+1)-np.log(actual+1)) ,2)) )
df1 = order_sub(df1)
 df2 = order_sub(df2 )<prepare_output>
train_sub = train[['id','place','date','ConfirmedCases','Fatalities']]
train_sub['logConfirmedCases'] = np.log(train_sub['ConfirmedCases'])
 train_sub = train_sub.set_index('date' )
BLEND=df1.copy()
BLEND.iloc[:,1:] = df1.iloc[:,1:]*0.60 +df2.iloc[:,1:]*0.40<define_variables>
list= []
for place in train_sub.place.unique() :
a = train_sub[train_sub['place']==place]
a['z_cases'] =(a['logConfirmedCases']- a['logConfirmedCases'].rolling(window=3 ).mean())/a['logConfirmedCases'].rolling(window=3 ).std()
a['zp_cases']= a['z_cases']- a['z_cases'].shift(3)
 a['z_death'] =(a['Fatalities']-a['Fatalities'].rolling(window=3 ).mean())/a['Fatalities'].rolling(window=3 ).std()
a['zp_death']= a['z_death']- a['z_death'].shift(3)
 list.append(a)
 rolling_df = pd.concat(list )
TARGET_COL = ['5-alpha_reductase_inhibitor', '11-beta-hsd1_inhibitor', 'acat_inhibitor', 'acetylcholine_receptor_agonist', 'acetylcholine_receptor_antagonist', 'acetylcholinesterase_inhibitor', 'adenosine_receptor_agonist', 'adenosine_receptor_antagonist', 'adenylyl_cyclase_activator', 'adrenergic_receptor_agonist', 'adrenergic_receptor_antagonist', 'akt_inhibitor', 'aldehyde_dehydrogenase_inhibitor', 'alk_inhibitor', 'ampk_activator', 'analgesic', 'androgen_receptor_agonist', 'androgen_receptor_antagonist', 'anesthetic_-_local', 'angiogenesis_inhibitor', 'angiotensin_receptor_antagonist', 'anti-inflammatory', 'antiarrhythmic', 'antibiotic', 'anticonvulsant', 'antifungal', 'antihistamine', 'antimalarial', 'antioxidant', 'antiprotozoal', 'antiviral', 'apoptosis_stimulant', 'aromatase_inhibitor', 'atm_kinase_inhibitor', 'atp-sensitive_potassium_channel_antagonist', 'atp_synthase_inhibitor', 'atpase_inhibitor', 'atr_kinase_inhibitor', 'aurora_kinase_inhibitor', 'autotaxin_inhibitor', 'bacterial_30s_ribosomal_subunit_inhibitor', 'bacterial_50s_ribosomal_subunit_inhibitor', 'bacterial_antifolate', 'bacterial_cell_wall_synthesis_inhibitor', 'bacterial_dna_gyrase_inhibitor', 'bacterial_dna_inhibitor', 'bacterial_membrane_integrity_inhibitor', 'bcl_inhibitor', 'bcr-abl_inhibitor', 'benzodiazepine_receptor_agonist', 'beta_amyloid_inhibitor', 'bromodomain_inhibitor', 'btk_inhibitor', 'calcineurin_inhibitor', 'calcium_channel_blocker', 'cannabinoid_receptor_agonist', 'cannabinoid_receptor_antagonist', 'carbonic_anhydrase_inhibitor', 'casein_kinase_inhibitor', 'caspase_activator', 'catechol_o_methyltransferase_inhibitor', 'cc_chemokine_receptor_antagonist', 'cck_receptor_antagonist', 'cdk_inhibitor', 'chelating_agent', 'chk_inhibitor', 'chloride_channel_blocker', 'cholesterol_inhibitor', 'cholinergic_receptor_antagonist', 'coagulation_factor_inhibitor', 'corticosteroid_agonist', 'cyclooxygenase_inhibitor', 'cytochrome_p450_inhibitor', 'dihydrofolate_reductase_inhibitor', 'dipeptidyl_peptidase_inhibitor', 'diuretic', 'dna_alkylating_agent', 'dna_inhibitor', 'dopamine_receptor_agonist', 'dopamine_receptor_antagonist', 'egfr_inhibitor', 'elastase_inhibitor', 'erbb2_inhibitor', 'estrogen_receptor_agonist', 'estrogen_receptor_antagonist', 'faah_inhibitor', 'farnesyltransferase_inhibitor', 'fatty_acid_receptor_agonist', 'fgfr_inhibitor', 'flt3_inhibitor', 'focal_adhesion_kinase_inhibitor', 'free_radical_scavenger', 'fungal_squalene_epoxidase_inhibitor', 'gaba_receptor_agonist', 'gaba_receptor_antagonist', 'gamma_secretase_inhibitor', 'glucocorticoid_receptor_agonist', 'glutamate_inhibitor', 'glutamate_receptor_agonist', 'glutamate_receptor_antagonist', 'gonadotropin_receptor_agonist', 'gsk_inhibitor', 'hcv_inhibitor', 'hdac_inhibitor', 'histamine_receptor_agonist', 'histamine_receptor_antagonist', 'histone_lysine_demethylase_inhibitor', 'histone_lysine_methyltransferase_inhibitor', 'hiv_inhibitor', 'hmgcr_inhibitor', 'hsp_inhibitor', 'igf-1_inhibitor', 'ikk_inhibitor', 'imidazoline_receptor_agonist', 'immunosuppressant', 'insulin_secretagogue', 'insulin_sensitizer', 'integrin_inhibitor', 'jak_inhibitor', 'kit_inhibitor', 'laxative', 'leukotriene_inhibitor', 'leukotriene_receptor_antagonist', 'lipase_inhibitor', 'lipoxygenase_inhibitor', 'lxr_agonist', 'mdm_inhibitor', 'mek_inhibitor', 'membrane_integrity_inhibitor', 'mineralocorticoid_receptor_antagonist', 'monoacylglycerol_lipase_inhibitor', 'monoamine_oxidase_inhibitor', 'monopolar_spindle_1_kinase_inhibitor', 'mtor_inhibitor', 'mucolytic_agent', 'neuropeptide_receptor_antagonist', 'nfkb_inhibitor', 'nicotinic_receptor_agonist', 'nitric_oxide_donor', 'nitric_oxide_production_inhibitor', 'nitric_oxide_synthase_inhibitor', 'norepinephrine_reuptake_inhibitor', 'nrf2_activator', 'opioid_receptor_agonist', 'opioid_receptor_antagonist', 'orexin_receptor_antagonist', 'p38_mapk_inhibitor', 'p-glycoprotein_inhibitor', 'parp_inhibitor', 'pdgfr_inhibitor', 'pdk_inhibitor', 'phosphodiesterase_inhibitor', 'phospholipase_inhibitor', 'pi3k_inhibitor', 'pkc_inhibitor', 'potassium_channel_activator', 'potassium_channel_antagonist', 'ppar_receptor_agonist', 'ppar_receptor_antagonist', 'progesterone_receptor_agonist', 'progesterone_receptor_antagonist', 'prostaglandin_inhibitor', 'prostanoid_receptor_antagonist', 'proteasome_inhibitor', 'protein_kinase_inhibitor', 'protein_phosphatase_inhibitor', 'protein_synthesis_inhibitor', 'protein_tyrosine_kinase_inhibitor', 'radiopaque_medium', 'raf_inhibitor', 'ras_gtpase_inhibitor', 'retinoid_receptor_agonist', 'retinoid_receptor_antagonist', 'rho_associated_kinase_inhibitor', 'ribonucleoside_reductase_inhibitor', 'rna_polymerase_inhibitor', 'serotonin_receptor_agonist', 'serotonin_receptor_antagonist', 'serotonin_reuptake_inhibitor', 'sigma_receptor_agonist', 'sigma_receptor_antagonist', 'smoothened_receptor_antagonist', 'sodium_channel_inhibitor', 'sphingosine_receptor_agonist', 'src_inhibitor', 'steroid', 'syk_inhibitor', 'tachykinin_antagonist', 'tgf-beta_receptor_inhibitor', 'thrombin_inhibitor', 'thymidylate_synthase_inhibitor', 'tlr_agonist', 'tlr_antagonist', 'tnf_inhibitor', 'topoisomerase_inhibitor', 'transient_receptor_potential_channel_antagonist', 'tropomyosin_receptor_kinase_inhibitor', 'trpv_agonist', 'trpv_antagonist', 'tubulin_inhibitor', 'tyrosine_kinase_inhibitor', 'ubiquitin_specific_protease_inhibitor', 'vegfr_inhibitor', 'vitamin_b', 'vitamin_d_receptor_agonist', 'wnt_inhibitor']
NUM_TARGET = len(TARGET_COL )<load_from_csv>
stationary_data =[]
for place in train_sub.place.unique() :
a= rolling_df[(rolling_df['place']==place)&(rolling_df['logConfirmedCases'] > 0)]['logConfirmedCases'].dropna()
try:
dftest = adfuller(a, autolag='AIC')
 if(dftest[1] < 0.001):
stationary_data.append(place)
 else:
pass
except:
pass
print(len(stationary_data))
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 df = pd.read_csv("/kaggle/input/lish-moa/sample_submission.csv")
 public_id = list(df['sig_id'].values)
 df_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_id = list(df_test['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=TARGET_COL)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_predict = BLEND.copy()
df_submit.loc[df_predict.sig_id,:] = df_predict[TARGET_COL].values
df_submit.loc[df_test[df_test.cp_type=='ctl_vehicle'].sig_id]= 0
df_submit.to_csv('submission.csv',index=True )<set_options>
station_death_data =[]
for place in train_sub.place.unique() :
dftest = adfuller(rolling_df[rolling_df['place']==place]['Fatalities'], autolag='AIC')
 if(dftest[1] < 0.001):
station_death_data.append(place)
 else:
pass
print(len(station_death_data))
%matplotlib inline
warnings.filterwarnings('ignore')
<load_from_csv>
confirmedc_lag = ['Anhui_China', 'Chongqing_China','Guangdong_China',
'Guizhou_China', 'Hainan_China', 'Hebei_China','Hubei_China',
'Ningxia_China','Shandong_China','Shanxi_China', 'Sichuan_China']
data = pd.read_csv(".. /input/bioresponse/train.csv")
 print(data.head(10))<count_missing_values>
fatalities_lag = ['Hubei_China']
print("Missing Values",data.isnull().sum().sum())
 print("Column names",data.columns )<create_dataframe>
non_stationary_death_data = [ele for ele in allplaces]
for place in fatalities_lag:
if place in allplaces:
non_stationary_death_data.remove(place)
 print(len(non_stationary_death_data))
Scaleddata = MinMaxScaler().fit_transform(data)
 data = pd.DataFrame(Scaleddata,columns=data.columns )<load_from_csv>
train_sub['logConfirmedCases']= train_sub['logConfirmedCases'].replace(to_replace=-inf,
value=0 )
train = pd.read_csv(".. /input/bioresponse/train.csv")
 test = pd.read_csv(".. /input/bioresponse/test.csv")
 train['label'] = 0
test['label'] = 1
training = train.drop('Activity',axis=1)
 combine = training.append(test)
 y =combine['label']
combine.drop(columns = ['label'],inplace=True)
 model = LGBMClassifier(n_estimators = 50)
 drop_list = []
for col in combine.columns:
score = cross_val_score(model,pd.DataFrame(combine[col]),y,cv=2,scoring='roc_auc')
 if(np.mean(score)> 0.7):
drop_list.append(col)
 print("Column with covariate shift:", col )<define_variables>
poly_data = train[['date','place',
'ConfirmedCases','Fatalities']].merge(test[['date','place']],
how='outer',
on=['date','place'] ).sort_values(['place',
'date'])
 print(poly_data.date.min() , test.date.min() , train.date.max() , poly_data.date.max() )
if len(drop_list)==0:
print("No presence of covariate shift" )<load_from_csv>
label = []
for place in poly_data.place.unique() :
labelrange = range(1,len(poly_data[poly_data['place']==place])+1)
 label.append([i for i in labelrange])
 lab = [item for lab in label for item in lab]
poly_data['label'] = lab
poly_data.head()
data = pd.read_csv(".. /input/bioresponse/train.csv")
 X = data
X = X.drop(columns='Activity')
 y = data['Activity']
y = y.values
y = y.reshape(( len(y), 1))
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.2,random_state=32)
 columns = X_train.columns
for col in columns:
MinMax = MinMaxScaler()
X_train_arr = X_train[col].astype(float ).values
X_test_arr = X_test[col].astype(float ).values
X_train_arr = MinMax.fit_transform(X_train_arr.reshape(-1,1))
X_test_arr = MinMax.transform(X_test_arr.reshape(-1,1))
X_train[col] = X_train_arr
X_test[col] = X_test_arr 
<init_hyperparams>
XYtrain['intercept']= -1
result=pd.DataFrame()
for place in poly_data.place.unique() :
for degree in [2,3,4,5,6]:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['ConfirmedCases']
model = make_pipeline(PolynomialFeatures(degree), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(features))
rmsle = RMSLE(y_pred, target)
 result = result.append(pd.DataFrame({'place':[place],
'degree':[degree],'RMSLE': [rmsle]}))
<choose_model_class>
best_degree = pd.DataFrame()
for place in result.place.unique() :
a = result[result['place']==place]
best_degree = best_degree.append(a[a['RMSLE'] == a['RMSLE'].min() ])
 print(best_degree.groupby('degree')['place'].nunique())
 print('Zero polynomial(no fit): ',best_degree[best_degree['RMSLE']<0.00001]['place'].unique() )
model = LGBMClassifier(random_state=314, silent=True, n_jobs=2,subsample_freq = 20, subsample = 0.9,
reg_lambda = 1.2, reg_alpha = 1.1,num_leaves= 200, n_estimators = 700,
min_split_gain = 0.4, max_depth = 15, colsample_bytree = 0.8 )<predict_on_test>
fit_best_degree = best_degree[best_degree['RMSLE']>0.00001]
twodeg_places = fit_best_degree[fit_best_degree['degree']==2]['place'].unique()
threedeg_places = fit_best_degree[fit_best_degree['degree']==3]['place'].unique()
fourdeg_places = fit_best_degree[fit_best_degree['degree']==4]['place'].unique()
fivedeg_places = fit_best_degree[fit_best_degree['degree']==5]['place'].unique()
sdeg_places = fit_best_degree[fit_best_degree['degree']==6]['place'].unique()
nofit_places1 = best_degree[best_degree['RMSLE']<0.00001]['place'].unique()
print(fit_best_degree.nunique())
 print(len(twodeg_places), len(threedeg_places),
len(fourdeg_places), len(fivedeg_places), len(sdeg_places), len(nofit_places1))
model.fit(X_train, y_train)
 pred = model.predict_proba(X_test)
 print("Log Loss Probability: ",log_loss(y_test,pred))<load_from_csv>
XYtest = XYtest.reset_index(drop=True)
 XYtest['intercept'] = -1
X_train = pd.read_csv(".. /input/bioresponse/train.csv")
 X_test = pd.read_csv(".. /input/bioresponse/test.csv")
 y_train = X_train["Activity"]
X_train.drop(columns=["Activity"],inplace=True)
 columns = X_train.columns
for col in columns:
MinMax = MinMaxScaler()
X_train_arr = X_train[col].astype(float ).values
X_test_arr = X_test[col].astype(float ).values
X_train_arr = MinMax.fit_transform(X_train_arr.reshape(-1,1))
X_test_arr = MinMax.transform(X_test_arr.reshape(-1,1))
X_train[col] = X_train_arr
X_test[col] = X_test_arr<choose_model_class>
poly_predicted_confirmedcases = pd.DataFrame()
for place in twodeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['ConfirmedCases']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(2), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
a = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred),columns=['place','ConfirmedCases'])
 poly_predicted_confirmedcases = poly_predicted_confirmedcases.append(a)
 for place in threedeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['ConfirmedCases']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(3), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
b = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','ConfirmedCases'])
 poly_predicted_confirmedcases = poly_predicted_confirmedcases.append(b)
 for place in fourdeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['ConfirmedCases']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(4), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
c = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','ConfirmedCases'])
 poly_predicted_confirmedcases = poly_predicted_confirmedcases.append(c)
 for place in fivedeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['ConfirmedCases']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(5), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
d = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','ConfirmedCases'])
 poly_predicted_confirmedcases = poly_predicted_confirmedcases.append(d)
 for place in sdeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['ConfirmedCases']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(6), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
e = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','ConfirmedCases'])
 poly_predicted_confirmedcases = poly_predicted_confirmedcases.append(e)
model = LGBMClassifier(random_state=314, silent=True, n_jobs=2,subsample_freq = 20, subsample = 0.9,
reg_lambda = 1.2, reg_alpha = 1.1,num_leaves= 200, n_estimators = 700,
min_split_gain = 0.4, max_depth = 15, colsample_bytree = 0.8 )<train_model>
fatalities_result=pd.DataFrame()
for place in poly_data.place.unique() :
for degree in [2,3,4,5,6]:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
model = make_pipeline(PolynomialFeatures(degree), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(features))
rmsle = RMSLE(y_pred, target)
 fatalities_result = fatalities_result.append(pd.DataFrame({'place':[place],
'degree':[degree],'RMSLE': [rmsle]}))
model.fit(X_train,y_train )<predict_on_test>
fat_best_degree = pd.DataFrame()
for place in fatalities_result.place.unique() :
a = fatalities_result[fatalities_result['place']==place]
fat_best_degree = fat_best_degree.append(a[a['RMSLE'] == a['RMSLE'].min() ])
 print(fat_best_degree.groupby('degree')['place'].nunique())
 print('Zero polynomial(no fit): ',
fat_best_degree[fat_best_degree['RMSLE']<0.000001]['place'].unique() )
predicted_prob = model.predict_proba(X_test )<define_variables>
fit_best_degree = fat_best_degree[fat_best_degree['RMSLE']>0.000001]
twodeg_places = fit_best_degree[fit_best_degree['degree']==2]['place'].unique()
threedeg_places = fit_best_degree[fit_best_degree['degree']==3]['place'].unique()
fourdeg_places = fit_best_degree[fit_best_degree['degree']==4]['place'].unique()
fivedeg_places = fit_best_degree[fit_best_degree['degree']==5]['place'].unique()
sevdeg_places = fit_best_degree[fit_best_degree['degree']==6]['place'].unique()
nofit_places2 = fat_best_degree[fat_best_degree['RMSLE']<0.000001]['place'].unique()
print(fit_best_degree.nunique())
 print(len(twodeg_places), len(threedeg_places),
len(fourdeg_places), len(fivedeg_places), len(sevdeg_places), len(nofit_places2))
Probability = predicted_prob[:,1]
MoleculeId = np.array(range(1,len(X_test)+1))<create_dataframe>
poly_predicted_fatalities = pd.DataFrame()
for place in twodeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(2), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
a = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(a)
 for place in threedeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(3), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
b = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(b)
 for place in fourdeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(4), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
c = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(c)
 for place in fivedeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(5), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
d = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(d)
 for place in sevdeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(6), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
e = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(e )
submission = pd.DataFrame()
submission["MoleculeId"] = MoleculeId
submission['PredictedProbability'] = Probability<save_to_csv>
for place in nofit_places1:
e = poly_data[(poly_data['place']==place)&(poly_data['date']>'2020-03-18')]
f = e['ConfirmedCases'].fillna(method = 'ffill')
 g = pd.DataFrame(zip(e['place'], f),columns=['place','ConfirmedCases'])
 poly_predicted_confirmedcases = poly_predicted_confirmedcases.append(g)
 for place in nofit_places2:
h = poly_data[(poly_data['place']==place)&(poly_data['date']>'2020-03-18')]
i = h['Fatalities'].fillna(method = 'ffill')
 j = pd.DataFrame(zip(h['place'], i),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(j )
submission.to_csv('submission.csv',index=None )<set_options>
poly_predicted_confirmedcases2= pd.DataFrame({'date':XYtest.date,
'place':poly_predicted_confirmedcases['place'].tolist() ,
'ConfirmedCases':poly_predicted_confirmedcases['ConfirmedCases'].tolist() })
 poly_predicted_confirmedcases2.head()
stop = set(stopwords.words('english'))
tf.logging.set_verbosity(tf.logging.ERROR)
 pd.options.display.max_rows = 10
pd.options.display.float_format = '{:.1f}'.format<load_from_csv>
poly_predicted_fatalities2= pd.DataFrame({'date':XYtest.date,
'place':poly_predicted_fatalities['place'].tolist() ,
'Fatalities':poly_predicted_fatalities['Fatalities'].tolist() })
 poly_predicted_fatalities2.head()
train = pd.read_csv(".. /input/train.csv", sep=",", parse_dates=['project_submitted_datetime'])
 resources = pd.read_csv(".. /input/resources.csv", sep=",")
 test = pd.read_csv(".. /input/test.csv", sep=",", parse_dates=['project_submitted_datetime'] )<merge>
poly_compiled = poly_predicted_confirmedcases2.merge(poly_predicted_fatalities2, how='inner', on=['place','date'] )
resources['cost'] = resources['quantity'] * resources['price']
resources_aggregated = resources.groupby('id' ).agg({'description': ['nunique'], 'quantity': ['sum'], 'cost': ['mean', 'sum']})
 resources_aggregated.columns = ['unique_items', 'total_quantity', 'mean_cost', 'total_cost']
resources_aggregated.reset_index(inplace=True)
 train = pd.merge(train, resources_aggregated, how='left', on='id')
 test = pd.merge(test, resources_aggregated, how='left', on='id' )<feature_engineering>
test_poly_compiled= test.merge(poly_compiled, how='inner', on=['place','date'])
 test_poly_compiled= test_poly_compiled.set_index('date')
 test_poly_compiled
train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] = train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] + ' ' + train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2']
train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2'] = train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_3'] + ' ' + train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_4']
train.drop(['project_essay_3', 'project_essay_4'], axis=1, inplace=True)
 test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] = test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] + ' ' + test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2']
test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2'] = test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_3'] + ' ' + test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_4']
test.drop(['project_essay_3', 'project_essay_4'], axis=1, inplace=True)
 train['project_essay_1'] = train['project_essay_1'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
train['project_essay_2'] = train['project_essay_2'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_essay_1'] = test['project_essay_1'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_essay_2'] = test['project_essay_2'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
train['project_resource_summary'] = train['project_resource_summary'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
train['project_title'] = train['project_title'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_resource_summary'] = test['project_resource_summary'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_title'] = test['project_title'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))<define_variables>
df_compiled = pd.DataFrame()
for place in test_poly_compiled.place.unique() :
a = test_poly_compiled[test_poly_compiled['place']==place]
ind_max_confirmedcases = np.argmax(a['ConfirmedCases'])
 a = a.replace(to_replace=a.loc[(a.index>ind_max_confirmedcases),'ConfirmedCases'].tolist() ,
value=a.loc[ind_max_confirmedcases,'ConfirmedCases'])
 ind_max_fatatities = np.argmax(a['Fatalities'])
 a = a.replace(to_replace=a.loc[(a.index>ind_max_fatatities),'Fatalities'].tolist() ,
value=a.loc[ind_max_fatatities,'Fatalities'])
 df_compiled = df_compiled.append(a)
 df_compiled[df_compiled['place']=='_Zimbabwe'].tail()
def get_quantile_based_boundaries(feature_values, num_buckets):
boundaries = np.arange(1.0, num_buckets)/ num_buckets
quantiles = feature_values.quantile(boundaries)
 return [quantiles[q] for q in quantiles.keys() ]<feature_engineering>
submission= pd.read_csv(datapath+'submission.csv' )
bins = [0., 2., 10.,]
test["experience"] = np.digitize(test['teacher_number_of_previously_posted_projects'], bins=bins)
 quantiles = get_quantile_based_boundaries(test['unique_items'], 6)
 test["unique_items_binned"] = np.digitize(test['unique_items'], bins=quantiles)
 quantiles = get_quantile_based_boundaries(test['total_quantity'], 6)
 test["total_quantity_binned"] = np.digitize(test['total_quantity'], bins=quantiles)
 quantiles = get_quantile_based_boundaries(test['mean_cost'], 6)
 test["mean_cost_binned"] = np.digitize(test['mean_cost'], bins=quantiles)
 quantiles = get_quantile_based_boundaries(test['total_cost'], 6)
 test["total_cost_binned"] = np.digitize(test['total_cost'], bins=quantiles )<define_variables>
sub2 = submission[['ForecastId']].merge(df_compiled[['ForecastId','ConfirmedCases','Fatalities']],
how='left',on='ForecastId' )
def preprocess_features(df):
selected_features = df[
["project_subject_categories",
"project_subject_subcategories",
"project_title",
"project_resource_summary",
"project_essay_1",
"project_essay_2",
"experience",
"unique_items_binned",
"total_quantity_binned",
"mean_cost_binned",
"total_cost_binned",
"encoded_category",
"encoded_subcategory"]]
return selected_features
def preprocess_targets(df):
output_targets = df["project_is_approved"]
return output_targets<categorify>
sub2['ConfirmedCases'] = sub2['ConfirmedCases'].round(1)
 sub2['Fatalities'] = sub2['Fatalities'].round(1 ).abs()
def target_encode(trn_series=None,
tst_series=None,
target=None,
min_samples_leaf=1,
smoothing=1,
noise_level=0):
assert len(trn_series)== len(target)
 assert trn_series.name == tst_series.name
temp = pd.concat([trn_series, target], axis=1)
 averages = temp.groupby(by=trn_series.name)[target.name].agg(["mean", "count"])
 smoothing = 1 /(1 + np.exp(-(averages["count"] - min_samples_leaf)/ smoothing))
prior = target.mean()
averages[target.name] = prior *(1 - smoothing)+ averages["mean"] * smoothing
averages.drop(["mean", "count"], axis=1, inplace=True)
 ft_trn_series = pd.merge(
 trn_series.to_frame(trn_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=trn_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_trn_series.index = trn_series.index
ft_tst_series = pd.merge(
 tst_series.to_frame(tst_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=tst_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_tst_series.index = tst_series.index
return ft_trn_series, ft_tst_series<categorify>
sub2.to_csv('submission.csv', index=False )