Kaggle competition: Homework for Students
<SOS> metric: auc Kaggle data source: homework-for-students<create_dataframe>
%matplotlib inline
pd.options.display.max_rows = 1000
pd.options.display.max_columns = 200
DIR = '/kaggle/input/homework-for-students4plus'
num_present_gender_class_6 = dict(gender_class_group.get_group(( 'male',3)).age_group.value_counts())
 survived_6 = {}
for i in num_present_gender_class_6:
value = gender_class_group.get_group(( 'male',3)).loc[gender_class_group.get_group(( 'male',3)).age_group==i]['Survived']
survived_6[i] = sum(value)
 list_6 = [survived_6,num_present_gender_class_6]
gender_class_6 = pd.DataFrame(list_6, index = ['Survived','Total'] ).transpose()
gender_class_6<groupby>
df_train_ori, df_test_ori = None, None
df_train, df_test = None, None
df_holdout = None
df_train_linear, df_test_linear = None, None
sr_title_train, sr_title_test = None, None
sr_emp_title_train, sr_emp_title_test = None, None
tfidf_pca_title_train, tfidf_pca_title_test = None, None
tfidf_pca_emp_title_train, tfidf_pca_emp_title_test = None, None
tfidf_pca_kmeans_cluster_train, tfidf_pca_kmeans_cluster_test = None, None
tfidf_pca_logreg_prediction_train, tfidf_pca_logreg_prediction_test = None, None
records_pca_tsne_kmeans_cluster_train, records_pca_tsne_kmeans_cluster_test = None, None
recoeds_kmeans_cluster_train, recoeds_kmeans_cluster_test = None, None
print("Number of missing values for males",gender_class_group.get_group(( 'male',3)).Age.isnull().sum())
 print("Total for males",gender_class_group.get_group(( 'male',3)).PassengerId.count() )<count_unique_values>
cols_qualitative, cols_quantitative, cols_flg = [], [], []
cols_unnecessary = []
y_pred_final = None
y_preds_list_for_ensemble = []
score_cv_avr, score_holdout = None, None
cvscore_list_for_ensemble, holdoutscore_list_for_ensemble = [], []
lgbm_params = {'objective': 'binary', 'metric': 'auc', 'learning_rate': 0.05}
num_2 = train_data.Parch.unique()
cnt_3 = Counter()
for num in train_data.Parch:
cnt_3.update([num])
 print(cnt_3)
 for num in cnt_3:
num_survived = train_data.loc[train_data.Parch == num]["Survived"]
print(sum(num_survived))
rate_survived = sum(num_survived)/cnt_3[num]
print("% of {} parent/child survived".format(num), rate_survived)
<feature_engineering>
def get_cols_to_use() :
global cols_qualitative, cols_quantitative, cols_flg
return cols_qualitative+cols_quantitative+cols_flg
for i in range(len(train_data)) :
if np.isnan(train_data["Age"][i]):
if train_data["Sex"][i]=='female':
train_data.at[i,"Age"]= 80
else:
train_data.at[i,"Age"]=70
for i in range(len(test_data)) :
if np.isnan(test_data["Age"][i]):
if test_data["Sex"][i]=='female':
test_data.at[i,"Age"]= 80
else:
test_data.at[i,"Age"]=70<feature_engineering>
def load_merged_df(cache=True):
print('load_merged_df(cache={})'.format(cache))
global df_train, df_test, df_train_ori, df_test_ori
if df_train_ori is not None:
print(' - already done, loading cache.')
 df_train = df_train_ori.copy()
df_test = df_test_ori.copy()
return
else:
df_train_ori = pd.read_csv('{}/train.csv'.format(DIR))
df_test_ori = pd.read_csv('{}/test.csv'.format(DIR))
merge_other_tables()
df_train = df_train_ori.copy()
df_test = df_test_ori.copy()
if cache is False:
df_train_ori = None
df_test_ori = None
return
def merge_other_tables() :
global df_train_ori, df_test_ori
df_zip_code = pd.read_csv('{}/free-zipcode-database.csv'.format(DIR), dtype={'WorldRegion': str})
 df_zip_code = df_zip_code[['State', 'LocationType', 'TotalWages']]
df_zip_code = df_zip_code[~df_zip_code.duplicated(subset='State')]
df_train_merged = pd.merge(df_train_ori, df_zip_code, left_on='addr_state', right_on='State', how='left' ).drop(columns='State')
 df_test_merged = pd.merge(df_test_ori, df_zip_code, left_on='addr_state', right_on='State', how='left' ).drop(columns='State')
 del df_zip_code
df_state_gdp = pd.read_csv('{}/US_GDP_by_State.csv'.format(DIR))
df_state_latlong = pd.read_csv('{}/statelatlong.csv'.format(DIR))
df_state_latlong = df_state_latlong[['State', 'City']]
df_state_latlong.rename(columns={'State': 'StateAbbr'}, inplace=True)
 df_state_gdp_df_state_latlong = pd.merge(df_state_gdp, df_state_latlong, left_on='State', right_on='City', how='left' ).drop(columns='City')
 df_state_gdp_df_state_latlong = df_state_gdp_df_state_latlong[df_state_gdp_df_state_latlong['year'] == 2015]
del df_state_gdp, df_state_latlong
df_train_merged = pd.merge(df_train_merged, df_state_gdp_df_state_latlong, left_on='addr_state', right_on='StateAbbr', how='left' ).drop(columns=['State', 'StateAbbr', 'year'])
 df_test_merged = pd.merge(df_test_merged, df_state_gdp_df_state_latlong, left_on='addr_state', right_on='StateAbbr', how='left' ).drop(columns=['State', 'StateAbbr', 'year'])
 del df_state_gdp_df_state_latlong
df_train_ori = df_train_merged.rename(columns={col: 'EXTER:'+col for col in ['LocationType', 'TotalWages', 'State & Local Spending',
'Gross State Product', 'Real State Growth %', 'Population(million)']})
 df_test_ori = df_test_merged.rename(columns={col: 'EXTER:'+col for col in ['LocationType', 'TotalWages', 'State & Local Spending',
'Gross State Product', 'Real State Growth %', 'Population(million)']})
 del df_train_merged, df_test_merged
return
<feature_engineering>
def object2date(convert_target_col):
print('object2date('{}')'.format(convert_target_col))
global df_train, df_test
def replace_date_col(x):
return x.replace('Jan', '01-01' ).replace('Feb', '01-02' ).replace('Mar', '01-03' ).replace('Apr', '01-04' ).replace('May', '01-05' ).replace('Jun', '01-06' ).replace('Jul', '01-07' ).replace('Aug', '01-08' ).replace('Sep', '01-09' ).replace('Oct', '01-10' ).replace('Nov', '01-11' ).replace('Dec', '01-12')
 df_train[convert_target_col] = df_train[convert_target_col].fillna('01-01-1900')
 df_test[convert_target_col] = df_test[convert_target_col].fillna('01-01-1900')
 df_train[convert_target_col] = df_train[convert_target_col].map(replace_date_col)
 df_test[convert_target_col] = df_test[convert_target_col].map(replace_date_col)
 df_train[convert_target_col] = pd.to_datetime(df_train[convert_target_col], format='%d-%m-%Y')
 df_test[convert_target_col] = pd.to_datetime(df_test[convert_target_col], format='%d-%m-%Y')
 return
interaction_train_2 = train_data.Age/train_data.Pclass
interaction_test_2 = test_data.Age/test_data.Pclass
interaction_test_1 = test_data.SibSp+test_data.Parch
interaction_train_1 = train_data.SibSp+train_data.Parch 
<prepare_x_and_y>
def extract_recent_data(oldest_year):
print('extract_recent_data(oldest_year={})'.format(oldest_year))
global df_train, df_test
df_train = df_train[df_train['issue_d'] >= dt.datetime(oldest_year, 1, 1)]
df_test = df_test[df_test['issue_d'] >= dt.datetime(oldest_year, 1, 1)]
return
features = ['Sex','Age','Pclass']
X_train = train_data[features]
X_test = test_data[features]
y = train_data.Survived<prepare_output>
def ordinal2int(convert_target_col):
print('ordinal2int('{}')'.format(convert_target_col))
global df_train, df_test
convert_mae = sorted(df_train[convert_target_col].unique())
 convert_ato = list(range(len(convert_mae)) [::-1])
 df_train[convert_target_col] = df_train[convert_target_col].replace(convert_mae, convert_ato)
 df_train[convert_target_col] = df_train[convert_target_col].astype(int)
 df_test[convert_target_col] = df_test[convert_target_col].replace(convert_mae, convert_ato)
 df_test[convert_target_col] = df_test[convert_target_col].astype(int)
 return
X_train.insert(1,"interaction",interaction_train_1)
 X_train.insert(2,"interaction_1",interaction_train_2)
 X_test.insert(1,"interaction",interaction_test_1)
 X_test.insert(2,"interaction_1",interaction_test_2)
<categorify>
def emplength2int() :
print('emplength2int() ')
 global df_train, df_test
convert_target_col = 'emp_length'
df_train[convert_target_col] = df_train[convert_target_col].fillna('
df_test[convert_target_col] = df_test[convert_target_col].fillna('
convert_mae = ['
convert_ato = [-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
df_train[convert_target_col] = df_train[convert_target_col].replace(convert_mae, convert_ato)
 df_train[convert_target_col] = df_train[convert_target_col].astype(int)
 df_test[convert_target_col] = df_test[convert_target_col].replace(convert_mae, convert_ato)
 df_test[convert_target_col] = df_test[convert_target_col].astype(int)
 return
label_X_train = X_train.copy()
label_X_test = X_test.copy()
label_encoder = LabelEncoder()
label_X_train['Sex'] = label_encoder.fit_transform(X_train['Sex'])
 label_X_test['Sex'] = label_encoder.transform(X_test['Sex'])
<feature_engineering>
def object2str(convert_target_col):
print('object2str('{}')'.format(convert_target_col))
global df_train, df_test
df_train[convert_target_col] = df_train[convert_target_col].fillna('
df_train[convert_target_col] = df_train[convert_target_col].astype(str)
 df_test[convert_target_col] = df_test[convert_target_col].fillna('
df_test[convert_target_col] = df_test[convert_target_col].astype(str)
 return
label_X_test.tail()<split>
def preprocess_text(preprocess_target_col):
print('preprocess_text('{}')'.format(preprocess_target_col))
global df_train, df_test, sr_title_train, sr_title_test, sr_emp_title_train, sr_emp_title_test
def replace_text(x):
x = x.replace('\t', '' ).replace('!', '' ).replace('?', '' ).replace(':)', '' ).replace('&', '')\ 
.replace('/', '' ).replace('-', '' ).replace('.', '' ).replace('"', '' ).replace (
train_X, valid_X, y_train, y_valid = train_test_split(label_X_train,y,random_state =1 )<drop_column>
def preprocess_words_of_text(preprocess_target_col):
print('preprocess_words_of_text('{}')'.format(preprocess_target_col))
global df_train, df_test
def replace_text(x):
x = x.replace('my ', '' ).replace('bye ', '' ).replace('the ', '')
 x = x.replace('cards', 'card' ).replace('credit consolidation', 'credit card consolidation')\ 
.replace('payoff', 'pay off' ).replace('cc', 'credit card' ).replace('improvements', 'improvement')\ 
.replace('refi ', 'refinancing ' ).replace('refinance', 'refinancing' ).replace('consoildation', 'consolidation')\ 
.replace('considation', 'consolidation' ).replace('consolodation', 'consolidation')\ 
.replace('refy ', 'refinancing ' ).replace('bills', 'bill')
 x = re.sub('refi$', 'refinancing', x)
 x = re.sub('refy$', 'refinancing', x)
 return x
df_train[preprocess_target_col] = df_train[preprocess_target_col].map(replace_text)
 df_test[preprocess_target_col] = df_test[preprocess_target_col].map(replace_text)
 return
<compute_train_metric>
def zipcode2str() :
print('zipcode2str() ')
 global df_train, df_test
convert_target_col = 'zip_code'
df_train[convert_target_col] = df_train[convert_target_col].str.replace('xx', '')
 df_train[convert_target_col] = df_train[convert_target_col].astype(str)
 df_test[convert_target_col] = df_test[convert_target_col].str.replace('xx', '')
 df_test[convert_target_col] = df_test[convert_target_col].astype(str)
 return
model = LogisticRegression()
model.fit(train_X, y_train)
 predictions_val = model.predict(valid_X)
 mae = mean_absolute_error(y_valid,predictions_val)
 print(mae )<save_to_csv>
def fillna_text(fillna_target_col, add_flg=False):
print('fillna_text('{}')'.format(fillna_target_col), '+flg' if add_flg else '')
 global df_train, df_test, cols_quantitative
isnull_flgs_train =(df_train[fillna_target_col] == '
isnull_flgs_test =(df_test[fillna_target_col] == '
if add_flg:
col_isnull_flg = 'FLG:{}_isnull'.format(fillna_target_col)
 df_train[col_isnull_flg] = 0
df_train.loc[isnull_flgs_train, col_isnull_flg] = 1
df_test[col_isnull_flg] = 0
df_test.loc[isnull_flgs_test, col_isnull_flg] = 1
cols_quantitative.append(col_isnull_flg)
 return
predictions = model.predict(label_X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission.csv', index=False)
 print("Your submission was successfully saved!" )<set_options>
def fillna_numeric(fillna_target_col, fillna_by='median', val_min=None, val_max=None, add_flg=False):
print('fillna_numeric('{}', fillna_by={})'.format(fillna_target_col, fillna_by), '+flg' if add_flg else '')
 global df_train, df_test, cols_quantitative
isnull_flgs_train = df_train[fillna_target_col].isnull()
isnull_flgs_test = df_test[fillna_target_col].isnull()
if(type(fillna_by)is int)or(type(fillna_by)is float):
df_train.loc[isnull_flgs_train, fillna_target_col] = fillna_by
df_test.loc[isnull_flgs_test, fillna_target_col] = fillna_by
if fillna_by == 'random':
fill_by_train = random.choices(df_train.loc[~isnull_flgs_train, fillna_target_col].tolist() , k=len(df_train[isnull_flgs_train]))
fill_by_test = random.choices(df_test.loc[~isnull_flgs_test, fillna_target_col].tolist() , k=len(df_test[isnull_flgs_test]))
df_train.loc[isnull_flgs_train, fillna_target_col] = fill_by_train
df_test.loc[isnull_flgs_test, fillna_target_col] = fill_by_test
if fillna_by == 'minmax':
val_btwn_minmax_flgs = df_train[fillna_target_col].between(val_min, val_max)
 val_btwn_minmax = df_train.loc[val_btwn_minmax_flgs, fillna_target_col].tolist()
fill_by_train = random.choices(val_btwn_minmax, k=len(df_train[isnull_flgs_train]))
fill_by_test = random.choices(val_btwn_minmax, k=len(df_test[isnull_flgs_test]))
df_train.loc[isnull_flgs_train, fillna_target_col] = fill_by_train
df_test.loc[isnull_flgs_test, fillna_target_col] = fill_by_test
if fillna_by == 'median':
median = df_train[fillna_target_col].median()
df_train.loc[isnull_flgs_train, fillna_target_col] = median
df_test.loc[isnull_flgs_test, fillna_target_col] = median
if add_flg:
col_isnull_flg = 'FLG:{}_isnull'.format(fillna_target_col)
 df_train[col_isnull_flg] = 0
df_train.loc[isnull_flgs_train, col_isnull_flg] = 1
df_test[col_isnull_flg] = 0
df_test.loc[isnull_flgs_test, col_isnull_flg] = 1
cols_quantitative.append(col_isnull_flg)
 return
warnings.filterwarnings("ignore" )<load_from_csv>
def log_transformation(transform_target_col, remove_outlier=False, add_flg=False):
print('log_transformation('{}')'.format(transform_target_col), '+remove_outlier' if remove_outlier else '', '+flg' if add_flg else '')
 global df_train, df_test, cols_flg
df_train[transform_target_col] = df_train[transform_target_col].apply(np.log1p)
 df_test[transform_target_col] = df_test[transform_target_col].apply(np.log1p)
 if remove_outlier:
p01 = df_train[transform_target_col].quantile(0.01)
 p99 = df_train[transform_target_col].quantile(0.99)
 isoutlier_flgs_train =(( df_train[transform_target_col] < p01)|(df_train[transform_target_col] > p99))
isoutlier_flgs_test =(( df_test[transform_target_col] < p01)|(df_test[transform_target_col] > p99))
df_train[transform_target_col] = df_train[transform_target_col].clip(p01, p99)
 df_test[transform_target_col] = df_test[transform_target_col].clip(p01, p99)
 if add_flg:
col_isoutlier_flg = 'FLG:{}_isoutlier'.format(transform_target_col)
 df_train[col_isoutlier_flg] = 0
df_train.loc[isoutlier_flgs_train, col_isoutlier_flg] = 1
df_test[col_isoutlier_flg] = 0
df_test.loc[isoutlier_flgs_test, col_isoutlier_flg] = 1
update_cols(add_cols_flg=[col_isoutlier_flg])
 return
train = pd.read_csv('.. /input/covid19-global-forecasting-week-3/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-3/test.csv' )<count_values>
def binning_values(cut_target_col, interval):
print('cut_values('{}', interval={})'.format(cut_target_col, interval))
global df_train, df_test
if int(str(interval*1000 ).split('.')[1])!= 0:
print(' > error: interval is too small')
 return
a = 1
for i in range(3):
if interval*a < 1:
a *= 10
def cut(x):
return(x*a-(( x*a)%(interval*a)))/a
df_train[cut_target_col] = df_train[cut_target_col].map(cut)
 df_test[cut_target_col] = df_test[cut_target_col].map(cut)
 return
curr_date = train['Date'].max()
world_cum_confirmed = sum(train[train['Date'] == curr_date].ConfirmedCases)
 world_cum_fatal = sum(train[train['Date'] == curr_date].Fatalities)
 print('Number of Countires: ', len(train['Country_Region'].unique()))
print('End date in train dset: ', curr_date)
 print('Number of confirmed cases: ', world_cum_confirmed)
 print('Number of fatal cases: ', world_cum_fatal )<sort_values>
def rank_gauss_transformation(n_split=5):
print('rank_gauss_transformation(n_split={})'.format(n_split))
global df_train, df_test, cols_quantitative
def split_list(l, n):
for idx in range(0, len(l), n):
yield l[idx:idx + n]
for cols in list(split_list(cols_quantitative, n_split)) :
df_all = pd.concat([df_train[cols], df_test[cols]], axis=0)
 df_all[cols] = quantile_transform(df_all[cols], n_quantiles=100, random_state=0, output_distribution='normal')
 df_train_transformed = df_all.iloc[:df_train.shape[0], :]
df_test_transformed = df_all.iloc[df_train.shape[0]:, :]
del df_all
df_train = pd.concat([df_train.drop(columns=cols), df_train_transformed], axis=1)
 df_test = pd.concat([df_test.drop(columns=cols), df_test_transformed], axis=1)
 del df_train_transformed, df_test_transformed
return
top_country_c = train[train['Date'] == curr_date].groupby(['Date','Country_Region'] ).sum().sort_values(['ConfirmedCases'], ascending=False)
 top_country_c.head(20 )<sort_values>
def fillna_text(fillna_target_col, add_flg=False):
print('fillna_text('{}')'.format(fillna_target_col), '+flg' if add_flg else '')
 global df_train, df_test, cols_flg
df_train[fillna_target_col] = df_train[fillna_target_col].fillna('
df_test[fillna_target_col] = df_test[fillna_target_col].fillna('
isnull_flgs_train =(df_train[fillna_target_col] == '
isnull_flgs_test =(df_test[fillna_target_col] == '
if add_flg:
col_isnull_flg = 'FLG:{}_isnull'.format(fillna_target_col)
 df_train[col_isnull_flg] = 0
df_train.loc[isnull_flgs_train, col_isnull_flg] = 1
df_test[col_isnull_flg] = 0
df_test.loc[isnull_flgs_test, col_isnull_flg] = 1
update_cols(add_cols_flg=[col_isnull_flg])
 return
top_country_f = train[train['Date'] == curr_date].groupby(['Date','Country_Region'] ).sum().sort_values(['Fatalities'], ascending=False)
 top_country_f.head(20 )<feature_engineering>
def fillna_numeric(fillna_target_col, fillna_by='median', val_min=None, val_max=None, add_flg=False):
print('fillna_numeric('{}', fillna_by={})'.format(fillna_target_col, fillna_by), '+flg' if add_flg else '')
 global df_train, df_test, cols_flg
isnull_flgs_train = df_train[fillna_target_col].isnull()
isnull_flgs_test = df_test[fillna_target_col].isnull()
if(type(fillna_by)is int)or(type(fillna_by)is float):
df_train.loc[isnull_flgs_train, fillna_target_col] = fillna_by
df_test.loc[isnull_flgs_test, fillna_target_col] = fillna_by
if fillna_by == 'random':
fill_by_train = random.choices(df_train.loc[~isnull_flgs_train, fillna_target_col].tolist() , k=len(df_train[isnull_flgs_train]))
fill_by_test = random.choices(df_test.loc[~isnull_flgs_test, fillna_target_col].tolist() , k=len(df_test[isnull_flgs_test]))
df_train.loc[isnull_flgs_train, fillna_target_col] = fill_by_train
df_test.loc[isnull_flgs_test, fillna_target_col] = fill_by_test
if fillna_by == 'minmax':
val_btwn_minmax_flgs = df_train[fillna_target_col].between(val_min, val_max)
 val_btwn_minmax = df_train.loc[val_btwn_minmax_flgs, fillna_target_col].tolist()
fill_by_train = random.choices(val_btwn_minmax, k=len(df_train[isnull_flgs_train]))
fill_by_test = random.choices(val_btwn_minmax, k=len(df_test[isnull_flgs_test]))
df_train.loc[isnull_flgs_train, fillna_target_col] = fill_by_train
df_test.loc[isnull_flgs_test, fillna_target_col] = fill_by_test
if fillna_by == 'median':
df_train.loc[isnull_flgs_train, fillna_target_col] = df_train[fillna_target_col].median()
df_test.loc[isnull_flgs_test, fillna_target_col] = df_test[fillna_target_col].median()
if add_flg:
col_isnull_flg = 'FLG:{}_isnull'.format(fillna_target_col)
 df_train[col_isnull_flg] = 0
df_train.loc[isnull_flgs_train, col_isnull_flg] = 1
df_test[col_isnull_flg] = 0
df_test.loc[isnull_flgs_test, col_isnull_flg] = 1
update_cols(add_cols_flg=[col_isnull_flg])
 return
train['MortalityRate'] = train['Fatalities'] / train['ConfirmedCases']
train['MortalityRate'] = train['MortalityRate'].fillna(0.0 )<sort_values>
def update_cols(add_cols_qualitative=[], add_cols_quantitative=[], add_cols_flg=[], add_cols_unnecessary=[], show_detail=False):
global df_train, df_test, cols_qualitative, cols_quantitative, cols_flg, cols_unnecessary
if show_detail:
print(' - update_cols(add_cols_qualitative={}, add_cols_quantitative={}, add_cols_flg={}, add_cols_unnecessary={})'.format(add_cols_qualitative, add_cols_quantitative, add_cols_flg, add_cols_unnecessary))
if add_cols_qualitative != []:
cols_qualitative = sorted(list(set(cols_qualitative)| set(add_cols_qualitative)))
 if add_cols_quantitative != []:
cols_quantitative = sorted(list(set(cols_quantitative)| set(add_cols_quantitative)))
 if add_cols_flg != []:
cols_flg = sorted(list(set(cols_flg)| set(add_cols_flg)))
 if add_cols_unnecessary != []:
cols_unnecessary = sorted(list(set(cols_unnecessary)| set(add_cols_unnecessary)))
 cols_qualitative = [col for col in cols_qualitative if col not in cols_unnecessary]
cols_quantitative = [col for col in cols_quantitative if col not in cols_unnecessary]
cols_flg = [col for col in cols_flg if col not in cols_unnecessary]
if show_detail:
print(' - no use : {}'.format(cols_unnecessary))
print(' > category: {}'.format(cols_qualitative))
print(' > numeric : {}'.format(cols_quantitative))
print(' > flag : {}'.format(cols_flg))
return
top_country_m = train[train['Date'] == curr_date].groupby(['Country_Region'] ).sum().sort_values(['MortalityRate'], ascending=False)
 top_country_m.head(10 )<data_type_conversions>
def tfidf_pca(extraction_target_col, n_dim_tfidf=500, n_dim_pca=10):
print('tfidf_pca('{}', n_dim_tfidf={}, n_dim_pca={})'.format(extraction_target_col, n_dim_tfidf, n_dim_pca))
global df_train, df_test
global tfidf_pca_title_train, tfidf_pca_title_test, tfidf_pca_emp_title_train, tfidf_pca_emp_title_test
extraction_begin = time.time()
if extraction_target_col == 'title':
if tfidf_pca_title_train is not None:
print(' - already done, loading cache.')
 df_train = df_train.reset_index(drop=True)
 df_test = df_test.reset_index(drop=True)
 df_train = pd.concat([df_train, tfidf_pca_title_train], axis=1)
 df_test = pd.concat([df_test, tfidf_pca_title_test], axis=1)
 update_cols(add_cols_quantitative=tfidf_pca_title_train.columns.tolist())
 else:
print(' - run TfidfVectorizer')
 tfidfv = TfidfVectorizer(max_features=n_dim_tfidf, use_idf=True, analyzer='word', ngram_range=(1, 2))
tfidfv.fit_transform(df_train[extraction_target_col])
 f_text_train = tfidfv.transform(df_train[extraction_target_col] ).todense()
f_text_test = tfidfv.transform(df_test[extraction_target_col] ).todense()
del tfidfv
print(' - run PCA')
 pca = PCA(n_components=n_dim_pca)
 pca.fit(f_text_train)
 f_text_train = pca.transform(f_text_train)
 f_text_test = pca.transform(f_text_test)
 del pca
cols_tfidf_pca = ['{}_tfidf_pca_{}'.format(extraction_target_col, i)for i in range(n_dim_pca)]
tfidf_pca_title_train = pd.DataFrame(columns=cols_tfidf_pca, data=f_text_train)
 tfidf_pca_title_test = pd.DataFrame(columns=cols_tfidf_pca, data=f_text_test)
 tfidf_pca_title_train = tfidf_pca_title_train.reset_index(drop=True)
 tfidf_pca_title_test = tfidf_pca_title_test.reset_index(drop=True)
 del f_text_train, f_text_test
df_train = df_train.reset_index(drop=True)
 df_test = df_test.reset_index(drop=True)
 df_train = pd.concat([df_train, tfidf_pca_title_train], axis=1)
 df_test = pd.concat([df_test, tfidf_pca_title_test], axis=1)
 update_cols(add_cols_quantitative=tfidf_pca_title_train.columns.tolist())
 else:
if tfidf_pca_emp_title_train is not None:
print(' - already done, loading cache.')
 df_train = df_train.reset_index(drop=True)
 df_test = df_test.reset_index(drop=True)
 df_train = pd.concat([df_train, tfidf_pca_emp_title_train], axis=1)
 df_test = pd.concat([df_test, tfidf_pca_emp_title_test], axis=1)
 update_cols(add_cols_quantitative=tfidf_pca_emp_title_train.columns.tolist())
 else:
print(' - run TfidfVectorizer')
 tfidfv = TfidfVectorizer(max_features=n_dim_tfidf, use_idf=True, analyzer='word', ngram_range=(1, 2))
tfidfv.fit_transform(df_train[extraction_target_col])
 f_text_train = tfidfv.transform(df_train[extraction_target_col] ).todense()
f_text_test = tfidfv.transform(df_test[extraction_target_col] ).todense()
del tfidfv
print(' - run PCA')
 pca = PCA(n_components=n_dim_pca)
 pca.fit(f_text_train)
 f_text_train = pca.transform(f_text_train)
 f_text_test = pca.transform(f_text_test)
 del pca
cols_tfidf_pca = ['{}_tfidf_pca_{}'.format(extraction_target_col, i)for i in range(n_dim_pca)]
tfidf_pca_emp_title_train = pd.DataFrame(columns=cols_tfidf_pca, data=f_text_train)
 tfidf_pca_emp_title_test = pd.DataFrame(columns=cols_tfidf_pca, data=f_text_test)
 tfidf_pca_emp_title_train = tfidf_pca_emp_title_train.reset_index(drop=True)
 tfidf_pca_emp_title_test = tfidf_pca_emp_title_test.reset_index(drop=True)
 del f_text_train, f_text_test
df_train = df_train.reset_index(drop=True)
 df_test = df_test.reset_index(drop=True)
 df_train = pd.concat([df_train, tfidf_pca_emp_title_train], axis=1)
 df_test = pd.concat([df_test, tfidf_pca_emp_title_test], axis=1)
 update_cols(add_cols_quantitative=tfidf_pca_emp_title_train.columns.tolist())
 extraction_elapsed_time = time.time() -extraction_begin
extraction_elapsed_sec = int(extraction_elapsed_time%60)
 extraction_elapsed_min = int(( extraction_elapsed_time-extraction_elapsed_sec)/60)
 print(' > elapsed time: {}:{}'.format(extraction_elapsed_min, str(extraction_elapsed_sec ).rjust(2, '0')))
 return
for df in [test]:
df['Date'] = pd.to_datetime(df['Date'], infer_datetime_format=True)
 df['Date'] = pd.to_datetime(df['Date'], infer_datetime_format=True )<drop_column>
def year2decade(convert_target_col, remove_original=True):
print('year2decade({}, remove_original={})'.format(convert_target_col, remove_original))
global df_train, df_test, cols_qualitative, cols_quantitative
col_decade_year = '{}_decade_year'.format(convert_target_col)
 df_train[col_decade_year] = df_train[convert_target_col].dt.year.astype(int)
 df_train[col_decade_year] -= df_train[col_decade_year] % 10
df_test[col_decade_year] = df_test[convert_target_col].dt.year.astype(int)
 df_test[col_decade_year] -= df_test[col_decade_year] % 10
if remove_original:
update_cols(add_cols_quantitative=[col_decade_year], add_cols_unnecessary=[convert_target_col])
 else:
update_cols(add_cols_quantitative=[col_decade_year])
 return
missed = "NA"
def State(state, country):
if state == missed: return country
return state<data_type_conversions>
def adversarial_validation_lgb(rate_sampling=1.0, n_folds=3, plot_feature_importance=True, ignore_target_enc=True, ignore_txt=True):
print('adversarial_validation_lgb(rate_sampling={})'.format(rate_sampling))
global df_train, df_test
cols_to_use = get_cols_to_use()
if ignore_target_enc:
cols_to_use = [col for col in cols_to_use if 'TARGET' not in col]
if ignore_txt:
cols_to_use = [col for col in cols_to_use if 'TXT' not in col]
df_all = pd.concat([df_train[cols_to_use], df_test[cols_to_use]])
 df_all['test'] = np.concatenate([np.zeros(len(df_train)) , np.ones(len(df_test)) ])
 X_all = df_all.drop(columns=['test'] ).values
y_all = df_all['test'].values
df_subset = df_all.sample(n=int(len(df_all)*rate_sampling))
X_subset = df_subset.drop(columns=['test'] ).values
y_subset = df_subset['test'].values
scores = []
f_importances = []
skf = StratifiedKFold(n_splits=n_folds, random_state=81, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(skf.split(X_subset, y_subset)) :
fold_begin = time.time()
_X_train, _y_train = X_subset[train_ix], y_subset[train_ix]
_X_val, _y_val = X_subset[val_ix], y_subset[val_ix]
lgb_train = lgb.Dataset(_X_train, _y_train)
 lgb_eval = lgb.Dataset(_X_val, _y_val, reference=lgb_train)
 model = lgb.train(lgbm_params, lgb_train, valid_sets=lgb_eval, verbose_eval=False)
 _y_pred = model.predict(_X_val, num_iteration=model.best_iteration)
 _score = roc_auc_score(_y_val, _y_pred)
 scores.append(_score)
 elapsed_time = time.time() -fold_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' - CV Score of Fold {} is {:.5f}({}:{})'.format(i, _score, elapsed_min, str(elapsed_sec ).rjust(2, '0')))
 print(' > Average CV Score is {:.5f}'.format(np.mean(scores)))
 lgb_train = lgb.Dataset(X_all, y_all)
 model = lgb.train(lgbm_params, lgb_train)
 df_all['test-like'] = model.predict(X_all, num_iteration=model.best_iteration)
 df_train['test-like'] = df_all.loc[df_all['test'] == 0, 'test-like']
update_cols(add_cols_unnecessary=['test-like'])
 del df_all
if plot_feature_importance:
f_importance = model.feature_importance(importance_type='gain')
 df_importance = pd.DataFrame(f_importance, index=cols_to_use, columns=['importance'])
 df_importance = df_importance.sort_values('importance')
 plt.figure()
df_importance.plot.barh(figsize=(8, 20), title='Feature Importances of Adversarial Validation')
 return
for df in [train, test]:
df['Province_State'].fillna(missed, inplace=True)
 df['Province_State'] = df.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : State(x['Province_State'], x['Country_Region']), axis=1)
 df.loc[:, 'Date'] = df.Date.dt.strftime("%m%d")
 df["Date"] = df["Date"].astype(int )<categorify>
def count_encoding(encoding_target_cols, remove_original=True):
print('count_encoding({}, remove_original={})'.format(encoding_target_cols, remove_original))
global df_train, df_test
for col in encoding_target_cols:
col_encoded = 'COUNT:{}'.format(col)
 freq = df_train[col].value_counts()
df_train[col_encoded] = df_train[col].map(freq)
 df_train.loc[df_train[col].isnull() , col_encoded] = len(df_train[col].isnull())
 df_test[col_encoded] = df_test[col].map(freq)
 df_test.loc[df_test[col].isnull() , col_encoded] = len(df_test[col].isnull())
 if remove_original:
update_cols(add_cols_quantitative=[col_encoded], add_cols_unnecessary=[col])
 else:
update_cols(add_cols_quantitative=[col_encoded])
 return
label_encoder = preprocessing.LabelEncoder()
for df in [train, test]:
df['Country_Region'] = label_encoder.fit_transform(df['Country_Region'])
 df['Province_State'] = label_encoder.fit_transform(df['Province_State'] )<choose_model_class>
def target_encoding(encoding_target_cols, remove_original=True):
print('target_encoding({}, remove_original={})'.format(encoding_target_cols, remove_original))
global df_train, df_test
X_temp = df_train[encoding_target_cols+[COL_TARGET]]
y_train = df_train[COL_TARGET]
for col in encoding_target_cols:
col_encoded = 'TARGET:{}'.format(col)
 target_mean_all = X_temp.groupby(col)[COL_TARGET].mean()
df_test[col_encoded] = df_test[col].map(target_mean_all)
 skf = StratifiedKFold(n_splits=5, random_state=81, shuffle=True)
 encoded_train = pd.Series(np.zeros(len(df_train)) , index=df_train.index)
 for train_ix, val_ix in skf.split(X_temp, y_train):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
target_mean = X_train_.groupby([col])[COL_TARGET].mean()
encoded_train.iloc[val_ix] = X_val[col].map(target_mean)
 df_train[col_encoded] = encoded_train
if remove_original:
update_cols(add_cols_quantitative=[col_encoded], add_cols_unnecessary=[col])
 else:
update_cols(add_cols_quantitative=[col_encoded])
 return
def RF() :
model = RandomForestRegressor(n_estimators = 100)
 return model
def XGB() :
model = XGBRegressor(n_estimators=1300)
 return model
def LGBM() :
model = LGBMRegressor(iterations=2)
 return model<data_type_conversions>
def onehot_encoding(encoding_target_cols):
print('onehot_encoding({})'.format(encoding_target_cols))
global df_train, df_test
for col in encoding_target_cols:
ohe = ce.OneHotEncoder(cols=[col], handle_unknown='ignore', use_cat_names=True)
 ohe.fit(df_train[col])
 df_ohe_train = ohe.transform(df_train[col])
 df_ohe_test = ohe.transform(df_test[col])
 df_train = pd.concat([df_train, df_ohe_train], axis=1)
 df_test = pd.concat([df_test, df_ohe_test], axis=1)
 update_cols(add_cols_quantitative=df_ohe_train.columns.tolist() , add_cols_unnecessary=[col])
 return
sub = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in range(len(unique_countries)) :
current_country_train = train.loc[train['Country_Region'] == unique_countries[country]]
current_country_test = test.loc[test['Country_Region'] == unique_countries[country]]
features = ['Country_Region', 'Province_State', 'Date']
X_train = current_country_train[features].to_numpy()
y1_train = current_country_train[['ConfirmedCases']].to_numpy()
y2_train = current_country_train[['Fatalities']].to_numpy()
X_test = current_country_test[features].to_numpy()
y1_train = y1_train.reshape(-1)
 y2_train = y2_train.reshape(-1)
 model1 = XGB()
model1.fit(X_train, y1_train)
 res_cnf_cls = np.round(model1.predict(X_test))
model2 = XGB()
model2.fit(X_train, y2_train)
 res_fac = np.round(model2.predict(X_test))
current_country_test_Id = current_country_test.loc[:, 'ForecastId']
pred = pd.DataFrame({'ForecastId': current_country_test_Id, 'ConfirmedCases': res_cnf_cls, 'Fatalities': res_fac})
 sub = pd.concat([sub, pred], axis=0 )<save_to_csv>
def pertition_holdout(rate_holdout=0.2, use_test_like_records=True, use_recent_records=False):
print('pertition_holdout(rate_holdout={}, use_test_like_records={}, use_recent_records={})'.format(rate_holdout, use_test_like_records, use_recent_records))
global df_train, df_test, df_holdout, cols_qualitative, cols_quantitative
df_train = df_train.reset_index(drop=True)
 n_holdout = len(df_train)*rate_holdout
true_rate = len(df_train[df_train[COL_TARGET] == 1])/len(df_train)
 holdout_ix = None
if use_test_like_records:
df_tmp = df_train[['test-like', COL_TARGET]].copy()
df_tmp = df_tmp.sort_values('test-like', ascending=False)
 ix_true = df_tmp[df_tmp[COL_TARGET] == 1][:int(n_holdout*true_rate)].index.tolist()
ix_false = df_tmp[df_tmp[COL_TARGET] == 0][:int(n_holdout*(1-true_rate)) ].index.tolist()
holdout_ix = ix_true + ix_false
del df_tmp
elif use_recent_records:
holdout_ix = df_train[df_train['issue_d'] >= dt.datetime(2015, 1, 1)].index.tolist()
else:
ix_true = df_train[df_train[COL_TARGET] == 1].sample(n=int(n_holdout*true_rate)).index.tolist()
ix_false = df_train[df_train[COL_TARGET] == 0].sample(n=int(n_holdout*(1-true_rate)) ).index.tolist()
holdout_ix = ix_true + ix_false
df_holdout = df_train.iloc[holdout_ix].copy()
df_holdout = df_holdout.reset_index(drop=True)
 df_train = df_train.drop(index=holdout_ix)
 df_train = df_train.reset_index(drop=True)
 return
sub.ForecastId = sub.ForecastId.astype('int')
 sub.to_csv('submission.csv', index=False)
 sub<set_options>
def cross_validation_lgb(rate_sampling=1.0, n_folds=5, plot_feature_importance=True, print_topworst_features=True, n_top=10):
print('
cross_validation_lgb(rate_sampling={})'.format(rate_sampling))
global df_train, df_test
global score_cv_avr
cols_to_use = get_cols_to_use()
df_train_subset = df_train.sample(n=int(len(df_train)*rate_sampling))
X_train = df_train_subset[cols_to_use].values
y_train = df_train_subset[COL_TARGET].values
X_test = df_test[cols_to_use].values
del df_train_subset
scores = []
f_importances = []
skf = StratifiedKFold(n_splits=n_folds, random_state=81, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(skf.split(X_train, y_train)) :
fold_begin = time.time()
_X_train, _y_train = X_train[train_ix], y_train[train_ix]
_X_val, _y_val = X_train[val_ix], y_train[val_ix]
lgb_train = lgb.Dataset(_X_train, _y_train)
 lgb_eval = lgb.Dataset(_X_val, _y_val, reference=lgb_train)
 model = lgb.train(lgbm_params, lgb_train, valid_sets=lgb_eval, verbose_eval=False)
 _y_pred = model.predict(_X_val, num_iteration=model.best_iteration)
 _score = roc_auc_score(_y_val, _y_pred)
 scores.append(_score)
 if plot_feature_importance or print_topworst_features:
f_importances.append(model.feature_importance(importance_type='gain'))
elapsed_time = time.time() -fold_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' - CV Score of Fold {} is {:.5f}({}:{})'.format(i, _score, elapsed_min, str(elapsed_sec ).rjust(2, '0')))
 score_cv_avr = np.mean(scores)
 print(' > Average CV Score is {:.5f}'.format(score_cv_avr))
if plot_feature_importance:
mean_of_feature_importances = np.array(f_importances ).mean(axis=0)
 df_mean_importance_of_cv = pd.DataFrame(mean_of_feature_importances, index=cols_to_use, columns=['importance'])
 df_mean_importance_of_cv = df_mean_importance_of_cv.sort_values('importance')
 plt.figure()
df_mean_importance_of_cv.plot.barh(figsize=(8, 20), title='Feature Importances of Cross Validation')
 del df_mean_importance_of_cv
if print_topworst_features:
common_features_top_n = None
common_features_worst_n = None
for i in range(n_folds):
sr_importance_tmp = pd.Series(f_importances[i], index=cols_to_use)
 sr_importance_tmp = sr_importance_tmp.sort_values(ascending=False)
 features_top_n = sr_importance_tmp[:n_top].index
features_worst_n = sr_importance_tmp[::-1][:n_top].index
if common_features_top_n is None:
common_features_top_n = features_top_n
common_features_worst_n = features_worst_n
else:
common_features_top_n = list(set(common_features_top_n)& set(features_top_n))
common_features_worst_n = list(set(common_features_worst_n)& set(features_worst_n))
print('
Print top and worst features(all)')
 print(' - top {} features commonly in each fold:'.format(n_top))
print(' > {}'.format(common_features_top_n))
print(' - worst {} features commonly in each fold:'.format(n_top))
print(' > {}'.format(common_features_worst_n))
common_features_top_n = None
common_features_worst_n = None
for i in range(n_folds):
sr_importance_tmp = pd.Series(f_importances[i], index=cols_to_use)
 cols_except_target_enc = [col for col in cols_to_use if 'TARGET' not in col]
sr_importance_tmp = sr_importance_tmp[cols_except_target_enc]
sr_importance_tmp = sr_importance_tmp.sort_values(ascending=False)
 features_top_n = sr_importance_tmp[:n_top].index
features_worst_n = sr_importance_tmp[::-1][:n_top].index
if common_features_top_n is None:
common_features_top_n = features_top_n
common_features_worst_n = features_worst_n
else:
common_features_top_n = list(set(common_features_top_n)& set(features_top_n))
common_features_worst_n = list(set(common_features_worst_n)& set(features_worst_n))
print('
Print top and worst features(ignore target enc)')
 print(' - top {} features commonly in each fold(ignore target enc):'.format(n_top))
print(' > {}'.format(common_features_top_n))
print(' - worst {} features commonly in each fold(ignore target enc):'.format(n_top))
print(' > {}'.format(common_features_worst_n))
return
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
pd.set_option('display.max_columns', 99)
 pd.set_option('display.max_rows', 99)
<set_options>
def cross_validation_xgb(rate_sampling=1.0, n_folds=5):
print('
cross_validation_xgb(rate_sampling={})'.format(rate_sampling))
global df_train, df_test
global score_cv_avr
cols_to_use = get_cols_to_use()
df_train_subset = df_train.sample(n=int(len(df_train)*rate_sampling))
X_train = df_train_subset[cols_to_use].values
y_train = df_train_subset[COL_TARGET].values
X_test = df_test[cols_to_use].values
del df_train_subset
scores = []
skf = StratifiedKFold(n_splits=n_folds, random_state=81, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(skf.split(X_train, y_train)) :
fold_begin = time.time()
_X_train, _y_train = X_train[train_ix], y_train[train_ix]
_X_val, _y_val = X_train[val_ix], y_train[val_ix]
D_train = xgb.DMatrix(_X_train, label=_y_train)
 D_val = xgb.DMatrix(_X_val, label=_y_val)
 xgb_param = {'objective': 'binary:logistic', 'eval_metric': 'auc'}
model = xgb.train(xgb_param, D_train, verbose_eval=False)
 _y_pred = model.predict(D_val, ntree_limit=model.best_ntree_limit)
 _score = roc_auc_score(_y_val, _y_pred)
 scores.append(_score)
 elapsed_time = time.time() -fold_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' - CV Score of Fold {} is {:.5f}({}:{})'.format(i, _score, elapsed_min, str(elapsed_sec ).rjust(2, '0')))
 score_cv_avr = np.mean(scores)
 print(' > Average CV Score is {:.5f}'.format(score_cv_avr))
return
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
sns.set_palette(sns.color_palette('tab20', 20)) 
<load_from_csv>
def cross_validation(clf_name, rate_sampling=1.0, n_folds=5):
print('
cross_validation(clf_name='{}', rate_sampling={})'.format(clf_name, rate_sampling))
global df_train, df_test
global score_cv_avr
cols_to_use = get_cols_to_use()
df_train_subset = df_train.sample(n=int(len(df_train)*rate_sampling))
X_train = df_train_subset[cols_to_use].values
y_train = df_train_subset[COL_TARGET].values
X_test = df_test[cols_to_use].values
del df_train_subset
scores = []
skf = StratifiedKFold(n_splits=n_folds, random_state=81, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(skf.split(X_train, y_train)) :
fold_begin = time.time()
_X_train, _y_train = X_train[train_ix], y_train[train_ix]
_X_val, _y_val = X_train[val_ix], y_train[val_ix]
clf = None
if clf_name == 'LogisticRegression':
clf = LogisticRegression(C=0.01)
 if clf_name == 'RandomForest':
clf = RandomForestClassifier()
if clf_name == 'HistGradientBoosting':
clf = HistGradientBoostingClassifier()
if clf_name == 'NeuralNetwork':
clf = Sequential()
clf.add(Dense(32, activation='relu'))
clf.add(Dense(16, activation='relu'))
clf.add(Dense(1, activation='sigmoid'))
clf.compile(loss='binary_crossentropy', optimizer='adam', metrics=[AUC() ])
 _y_pred = None
if clf_name == 'NeuralNetwork':
clf.fit(_X_train, _y_train, epochs=5)
 _y_pred = clf.predict(_X_val)[:, 0]
else:
clf.fit(_X_train, _y_train)
 _y_pred = clf.predict_proba(_X_val)[:,1]
_score = roc_auc_score(_y_val, _y_pred)
 scores.append(_score)
 elapsed_time = time.time() -fold_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' - CV Score of Fold {} is {:.5f}({}:{})'.format(i, _score, elapsed_min, str(elapsed_sec ).rjust(2, '0')))
 score_cv_avr = np.mean(scores)
 print(' > Average CV Score is {:.5f}'.format(score_cv_avr))
return
COMP = '.. /input/covid19-global-forecasting-week-3'
DATEFORMAT = '%Y-%m-%d'
def get_comp_data(COMP):
train = pd.read_csv(f'{COMP}/train.csv')
 test = pd.read_csv(f'{COMP}/test.csv')
 submission = pd.read_csv(f'{COMP}/submission.csv')
 print(train.shape, test.shape, submission.shape)
 train['Country_Region'] = train['Country_Region'].str.replace(',', '')
 test['Country_Region'] = test['Country_Region'].str.replace(',', '')
 train['Location'] = train['Country_Region'] + '-' + train['Province_State'].fillna('')
 test['Location'] = test['Country_Region'] + '-' + test['Province_State'].fillna('')
 train['LogConfirmed'] = to_log(train.ConfirmedCases)
 train['LogFatalities'] = to_log(train.Fatalities)
 train = train.drop(columns=['Province_State'])
 test = test.drop(columns=['Province_State'])
 country_codes = pd.read_csv('.. /input/covid19-metadata/country_codes.csv', keep_default_na=False)
 train = train.merge(country_codes, on='Country_Region', how='left')
 test = test.merge(country_codes, on='Country_Region', how='left')
 train['DateTime'] = pd.to_datetime(train['Date'])
 test['DateTime'] = pd.to_datetime(test['Date'])
 return train, test, submission
def process_each_location(df):
dfs = []
for loc, df in tqdm(df.groupby('Location')) :
df = df.sort_values(by='Date')
 df['Fatalities'] = df['Fatalities'].cummax()
df['ConfirmedCases'] = df['ConfirmedCases'].cummax()
df['LogFatalities'] = df['LogFatalities'].cummax()
df['LogConfirmed'] = df['LogConfirmed'].cummax()
df['LogConfirmedNextDay'] = df['LogConfirmed'].shift(-1)
 df['ConfirmedNextDay'] = df['ConfirmedCases'].shift(-1)
 df['DateNextDay'] = df['Date'].shift(-1)
 df['LogFatalitiesNextDay'] = df['LogFatalities'].shift(-1)
 df['FatalitiesNextDay'] = df['Fatalities'].shift(-1)
 df['LogConfirmedDelta'] = df['LogConfirmedNextDay'] - df['LogConfirmed']
df['ConfirmedDelta'] = df['ConfirmedNextDay'] - df['ConfirmedCases']
df['LogFatalitiesDelta'] = df['LogFatalitiesNextDay'] - df['LogFatalities']
df['FatalitiesDelta'] = df['FatalitiesNextDay'] - df['Fatalities']
dfs.append(df)
 return pd.concat(dfs)
 def add_days(d, k):
return dt.datetime.strptime(d, DATEFORMAT)+ dt.timedelta(days=k)
 def to_log(x):
return np.log(x + 1)
 def to_exp(x):
return np.exp(x)- 1 
<count_duplicates>
def holdout_validation_lgb(plot_feature_importance=True, print_topworst_features=True, print_topworst_features_by_conditions=True):
print('
holdout_validation_lgb() ')
 global df_train, df_test, df_holdout
global score_holdout
validation_begin = time.time()
cols_to_use = get_cols_to_use()
X_train = df_train[cols_to_use].values
y_train = df_train[COL_TARGET].values
X_val = df_holdout[cols_to_use].values
y_val = df_holdout[COL_TARGET].values
lgb_train = lgb.Dataset(X_train, y_train)
 lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train)
 model = lgb.train(lgbm_params, lgb_train, valid_sets=lgb_eval, verbose_eval=False)
 y_pred = model.predict(X_val, num_iteration=model.best_iteration)
 score_holdout = roc_auc_score(y_val, y_pred)
 f_importance = model.feature_importance(importance_type='gain')
 elapsed_time = time.time() -validation_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' - Score of Holdout is {:.5f}({}:{})'.format(score_holdout, elapsed_min, str(elapsed_sec ).rjust(2, '0')))
 if plot_feature_importance:
df_mean_importance_of_cv = pd.DataFrame(f_importance, index=cols_to_use, columns=['importance'])
 df_mean_importance_of_cv = df_mean_importance_of_cv.sort_values('importance')
 plt.figure()
df_mean_importance_of_cv.plot.barh(figsize=(8, 20), title='Feature Importances of Holdout Validation')
 del df_mean_importance_of_cv
if print_topworst_features:
sr_importance_tmp = pd.Series(f_importance, index=cols_to_use)
 sr_importance_tmp = sr_importance_tmp.sort_values(ascending=False)
 print('
Features sorted by importance(all): {}'.format(sr_importance_tmp.index.tolist()))
if print_topworst_features_by_conditions:
conds = ['INTRCTN', 'DIFF', 'DIV']
for cond in conds:
sr_importance_tmp = pd.Series(f_importance, index=cols_to_use)
 sr_importance_tmp = sr_importance_tmp[sr_importance_tmp.index.str.startswith(cond)]
sr_importance_tmp = sr_importance_tmp.sort_values(ascending=False)
 print('
Features sorted by importance(including {}): {}'.format(cond, sr_importance_tmp.index.tolist()))
return
train[train.geo_region.isna() ].Country_Region.unique()
train = train.fillna('
test = test.fillna('
train[train.duplicated(['Date', 'Location'])]
train.count()<save_to_csv>
def holdout_validation_xgb() :
print('
holdout_validation_xgb() ')
 global df_train, df_test, df_holdout
global score_holdout
validation_begin = time.time()
cols_to_use = get_cols_to_use()
X_train = df_train[cols_to_use].values
y_train = df_train[COL_TARGET].values
X_val = df_holdout[cols_to_use].values
y_val = df_holdout[COL_TARGET].values
D_train = xgb.DMatrix(X_train, label=y_train)
 D_val = xgb.DMatrix(X_val, label=y_val)
 xgb_param = {'objective': 'binary:logistic', 'eval_metric': 'auc'}
model = xgb.train(xgb_param, D_train, verbose_eval=False)
 y_pred = model.predict(D_val, ntree_limit=model.best_ntree_limit)
 score_holdout = roc_auc_score(y_val, y_pred)
 elapsed_time = time.time() -validation_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' - Score of Holdout is {:.5f}({}:{})'.format(score_holdout, elapsed_min, str(elapsed_sec ).rjust(2, '0')))
 return
train = train.sort_values(by='Date')
 countries_latest_state = train[train['Date'] == TRAIN_END].groupby([
'Country_Region', 'continent', 'geo_region', 'country_iso_code_3'] ).sum() [[
'ConfirmedCases', 'Fatalities']].reset_index()
countries_latest_state['Log10Confirmed'] = np.log10(countries_latest_state.ConfirmedCases + 1)
 countries_latest_state['Log10Fatalities'] = np.log10(countries_latest_state.Fatalities + 1)
 countries_latest_state = countries_latest_state.sort_values(by='Fatalities', ascending=False)
 countries_latest_state.to_csv('countries_latest_state.csv', index=False)
 countries_latest_state.shape
countries_latest_state.head()<merge>
def holdout_validation(clf_name, plot_feature_importance=False):
print('
holdout_validation(clf_name='{}')'.format(clf_name))
global df_train, df_test, df_holdout
global score_holdout
validation_begin = time.time()
cols_to_use = get_cols_to_use()
X_train = df_train[cols_to_use].values
y_train = df_train[COL_TARGET].values
X_val = df_holdout[cols_to_use].values
y_val = df_holdout[COL_TARGET].values
clf = None
if clf_name == 'LogisticRegression':
clf = LogisticRegression(C=0.01)
 if clf_name == 'RandomForest':
clf = RandomForestClassifier()
if clf_name == 'HistGradientBoosting':
clf = HistGradientBoostingClassifier()
if clf_name == 'NeuralNetwork':
clf = Sequential()
clf.add(Dense(32, activation='relu'))
clf.add(Dense(16, activation='relu'))
clf.add(Dense(1, activation='sigmoid'))
clf.compile(loss='binary_crossentropy', optimizer='adam', metrics=[AUC() ])
 y_pred = None
if clf_name == 'NeuralNetwork':
clf.fit(X_train, y_train, epochs=5)
 y_pred = clf.predict(X_val)[:, 0]
else:
clf.fit(X_train, y_train)
 y_pred = clf.predict_proba(X_val)[:,1]
score_holdout = roc_auc_score(y_val, y_pred)
 if plot_feature_importance and(clf_name == 'LogisticRegression'):
sr_importance = pd.Series(clf.coef_[0], index=cols_to_use)
 sr_importance = sr_importance.abs()
sr_importance = sr_importance.sort_values()
plt.figure()
sr_importance.plot.barh(figsize=(8, 20), title='Importance of Logistic Regression Features')
 print(' - ranking of important features: {}'.format(sr_importance.index.tolist() [::-1]))
elapsed_time = time.time() -validation_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' - Score of Holdout is {:.5f}({}:{})'.format(score_holdout, elapsed_min, str(elapsed_sec ).rjust(2, '0')))
 return
latest_loc = train[train['Date'] == TRAIN_END][['Location', 'ConfirmedCases', 'Fatalities']]
max_loc = train.groupby(['Location'])[['ConfirmedCases', 'Fatalities']].max().reset_index()
check = pd.merge(latest_loc, max_loc, on='Location')
 np.mean(check.ConfirmedCases_x == check.ConfirmedCases_y)
 np.mean(check.Fatalities_x == check.Fatalities_y)
 check[check.Fatalities_x != check.Fatalities_y]
check[check.ConfirmedCases_x != check.ConfirmedCases_y]<feature_engineering>
def modeling_and_prediction_lgb(cross_validation_ensemble=False, n_folds=5):
print('modeling_and_prediction_lgb(cross_validation_ensemble={})'.format(cross_validation_ensemble))
global df_train, df_test, df_holdout
global y_pred_final
cols_to_use = get_cols_to_use()
df_train = pd.concat([df_train, df_holdout])
 X_train = df_train[cols_to_use].values
y_train = df_train[COL_TARGET].values
X_test = df_test[cols_to_use].values
if cross_validation_ensemble:
y_preds_sum = [0 for i in range(len(X_test)) ]
skf = StratifiedKFold(n_splits=n_folds, random_state=81, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(skf.split(X_train, y_train)) :
_X_train, _y_train = X_train[train_ix], y_train[train_ix]
_X_val, _y_val = X_train[val_ix], y_train[val_ix]
lgb_train = lgb.Dataset(_X_train, _y_train)
 model = lgb.train(lgbm_params, lgb_train, valid_sets=lgb_train, verbose_eval=False)
 y_preds_sum = [pr1+pr2 for(pr1, pr2)in zip(y_preds_sum,
model.predict(X_test, num_iteration=model.best_iteration)) ]
y_pred_final = [pr/n_folds for pr in y_preds_sum]
else:
lgb_train = lgb.Dataset(X_train, y_train)
 model = lgb.train(lgbm_params, lgb_train, valid_sets=lgb_train, verbose_eval=False)
 y_pred_final = model.predict(X_test, num_iteration=model.best_iteration)
 return
regional_progress = train_clean.groupby(['DateTime', 'continent'] ).sum() [['ConfirmedCases', 'Fatalities']].reset_index()
regional_progress['Log10Confirmed'] = np.log10(regional_progress.ConfirmedCases + 1)
 regional_progress['Log10Fatalities'] = np.log10(regional_progress.Fatalities + 1)
 regional_progress = regional_progress[regional_progress.continent != '<merge>
def modeling_and_prediction_xgb(cross_validation_ensemble=False, n_folds=5):
print('modeling_and_prediction_xgb(cross_validation_ensemble={})'.format(cross_validation_ensemble))
global df_train, df_test, df_holdout
global y_pred_final
cols_to_use = get_cols_to_use()
df_train = pd.concat([df_train, df_holdout])
 X_train = df_train[cols_to_use].values
y_train = df_train[COL_TARGET].values
X_test = df_test[cols_to_use].values
D_test = xgb.DMatrix(X_test)
 if cross_validation_ensemble:
y_preds_sum = [0 for i in range(len(X_test)) ]
skf = StratifiedKFold(n_splits=n_folds, random_state=81, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(skf.split(X_train, y_train)) :
_X_train, _y_train = X_train[train_ix], y_train[train_ix]
_X_val, _y_val = X_train[val_ix], y_train[val_ix]
D_train = xgb.DMatrix(_X_train, label=_y_train)
 xgb_param = {'objective': 'binary:logistic', 'eval_metric': 'auc'}
model = xgb.train(xgb_param, D_train, verbose_eval=False)
 y_preds_sum = [pr1+pr2 for(pr1, pr2)in zip(y_preds_sum,
model.predict(D_test, ntree_limit=model.best_ntree_limit)) ]
y_pred_final = y_preds_sum / n_folds
else:
D_train = xgb.DMatrix(X_train, label=y_train)
 xgb_param = {'objective': 'binary:logistic', 'eval_metric': 'auc'}
model = xgb.train(xgb_param, D_train, valid_sets=D_train, verbose_eval=False)
 y_pred_final = model.predict(D_test, ntree_limit=model.best_ntree_limit)
 return
countries_0301 = country_progress[country_progress.Date == '2020-03-01'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_0331 = country_progress[country_progress.Date == '2020-03-31'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_in_march = pd.merge(countries_0301, countries_0331, on='Country_Region', suffixes=['_0301', '_0331'])
 countries_in_march['IncreaseInMarch'] = countries_in_march.ConfirmedCases_0331 /(countries_in_march.ConfirmedCases_0301 + 1)
 countries_in_march = countries_in_march[countries_in_march.ConfirmedCases_0331 > 200].sort_values(
 by='IncreaseInMarch', ascending=False)
 countries_in_march.tail(15 )<save_to_csv>
def modeling_and_prediction(clf_name, cross_validation_ensemble=False, n_folds=5):
print('modeling_and_prediction(clf_name='{}', cross_validation_ensemble={})'.format(clf_name, cross_validation_ensemble))
global df_train, df_test, df_holdout
global y_pred_final
cols_to_use = get_cols_to_use()
df_train = pd.concat([df_train, df_holdout])
 X_train = df_train[cols_to_use].values
y_train = df_train[COL_TARGET].values
X_test = df_test[cols_to_use].values
if cross_validation_ensemble:
y_preds_sum = [0 for i in range(len(X_test)) ]
skf = StratifiedKFold(n_splits=n_folds, random_state=81, shuffle=True)
 for i,(train_ix, val_ix)in enumerate(skf.split(X_train, y_train)) :
_X_train, _y_train = X_train[train_ix], y_train[train_ix]
_X_val, _y_val = X_train[val_ix], y_train[val_ix]
clf = None
if clf_name == 'LogisticRegression':
clf = LogisticRegression(C=0.01)
 if clf_name == 'RandomForest':
clf = RandomForestClassifier()
if clf_name == 'HistGradientBoosting':
clf = HistGradientBoostingClassifier()
if clf_name == 'NeuralNetwork':
clf = Sequential()
clf.add(Dense(32, activation='relu'))
clf.add(Dense(16, activation='relu'))
clf.add(Dense(1, activation='sigmoid'))
clf.compile(loss='binary_crossentropy', optimizer='adam', metrics=[AUC() ])
 _y_pred = None
if clf_name == 'NeuralNetwork':
clf.fit(_X_train, _y_train, epochs=5)
 y_preds_sum = [pr1+pr2 for(pr1, pr2)in zip(y_preds_sum, clf.predict(X_test)[:, 0])]
else:
clf.fit(_X_train, _y_train)
 y_preds_sum = [pr1+pr2 for(pr1, pr2)in zip(y_preds_sum, clf.predict_proba(X_test)[:,1])]
y_pred_final = [pr/n_folds for pr in y_preds_sum]
else:
clf = None
if clf_name == 'LogisticRegression':
clf = LogisticRegression(C=0.01)
 if clf_name == 'RandomForest':
clf = RandomForestClassifier()
if clf_name == 'HistGradientBoosting':
clf = HistGradientBoostingClassifier()
if clf_name == 'NeuralNetwork':
clf = MLPClassifier()
clf.fit(X_train, y_train)
 y_pred_final = clf.predict_proba(X_test)[:,1]
return
train_clean['Geo
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_confirmed_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_confirmed_deltas = latest.merge(daily_confirmed_deltas, on='Geo
daily_confirmed_deltas.shape
daily_confirmed_deltas.head()
daily_confirmed_deltas.to_csv('daily_confirmed_deltas.csv', index=False )<save_to_csv>
cols_qualitative, cols_quantitative, cols_flg, cols_unnecessary = [], [], [], []
print('
')
 notebook_begin = time.time()
load_merged_df(cache=True)
 COL_ID = 'ID'
COL_TARGET = 'loan_condition'
print('
')
 print('Adding n_nan')
 df_train['n_nan'] = df_train.isnull().sum(axis=1)
 df_test['n_nan'] = df_test.isnull().sum(axis=1)
 object2date('issue_d')
 object2date('earliest_cr_line')
 extract_recent_data(oldest_year=2014)
 cols_unnecessary.append('issue_d')
 ordinal2int('grade')
 ordinal2int('sub_grade')
 emplength2int()
object2str('title')
 object2str('emp_title')
 preprocess_text('title')
 preprocess_words_of_text('title')
 zipcode2str()
print('
')
 load_cols_by_dtype()
print('
')
 fillna_text('title', add_flg=False)
 fillna_text('emp_title', add_flg=False)
 fillna_numeric('dti', fillna_by=999.0)
 fillna_numeric('mths_since_last_delinq', fillna_by='minmax', val_min=0, val_max=85, add_flg=True)
 fillna_numeric('mths_since_last_record', fillna_by='random', add_flg=True)
 fillna_numeric('revol_util', fillna_by='minmax', val_min=0, val_max=100, add_flg=True)
 fillna_numeric('mths_since_last_major_derog', fillna_by='minmax', val_min=0, val_max=90, add_flg=True)
 fillna_numeric('inq_last_6mths', fillna_by='median')
 fillna_numeric('EXTER:TotalWages', fillna_by='random', add_flg=True)
 print('
')
 log_transformation('loan_amnt', remove_outlier=False, add_flg=True)
 log_transformation('installment', remove_outlier=False, add_flg=True)
 log_transformation('annual_inc', remove_outlier=True, add_flg=True)
 log_transformation('revol_bal', remove_outlier=True, add_flg=True)
 log_transformation('tot_coll_amt', remove_outlier=True, add_flg=True)
 log_transformation('tot_cur_bal', remove_outlier=True, add_flg=True)
 log_transformation('EXTER:TotalWages', remove_outlier=False)
 print('
')
 tfidf_pca('emp_title', n_dim_tfidf=500, n_dim_pca=10)
 update_cols(add_cols_unnecessary=['title', 'emp_title'])
 year2decade('earliest_cr_line', remove_original=True)
 ordinal2int('earliest_cr_line_decade_year')
 print('
')
 update_cols(add_cols_unnecessary=['zip_code'])
 onehot_encoding(cols_qualitative)
 print('
')
 print('Scaling with Standard Scaler')
 for col in cols_quantitative:
scaler = StandardScaler()
scaler.fit(df_train[[col]])
 df_train[col] = scaler.transform(df_train[[col]])
 df_test[col] = scaler.transform(df_test[[col]])
 print('Copying Dataframe to df_linear')
 cols_to_use = get_cols_to_use()
df_train_linear = df_train[cols_to_use].copy()
df_test_linear = df_test[cols_to_use].copy()
print('number of features: {}'.format(len(get_cols_to_use())))
 print('
')
 print('number of features: {}'.format(len(get_cols_to_use())))
 pertition_holdout(rate_holdout=0.2, use_test_like_records=False, use_recent_records=True)
 cross_validation(clf_name='LogisticRegression', rate_sampling=1.0)
 print('
')
 holdout_validation(clf_name='LogisticRegression', plot_feature_importance=True)
 print('
')
 modeling_and_prediction(clf_name='LogisticRegression', cross_validation_ensemble=True, n_folds=5)
 print('Appending model predictions to y_preds_list')
 y_preds_list_for_ensemble.append(y_pred_final)
 cvscore_list_for_ensemble.append(score_cv_avr)
 holdoutscore_list_for_ensemble.append(score_holdout)
 print('
')
 print('number of features: {}'.format(len(get_cols_to_use())))
 print('Print scores')
 print(' > Avr of CV: {:.5f}'.format(score_cv_avr))
print(' > Holdout : {:.5f}'.format(score_holdout))
print('Calculating total elapsed time')
 elapsed_time = time.time() -notebook_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' > total elapsed time...{}:{}'.format(elapsed_min, str(elapsed_sec ).rjust(2, '0')))
deltas = train_clean[np.logical_and(
 train_clean.LogConfirmed > 2,
~train_clean.Location.str.startswith('China')
)].dropna().sort_values(by='LogConfirmedDelta', ascending=False)
 deltas['start'] = deltas['LogConfirmed'].round(0)
 confirmed_deltas = pd.concat([
deltas.groupby('start')[['LogConfirmedDelta']].mean() ,
deltas.groupby('start')[['LogConfirmedDelta']].std() ,
deltas.groupby('start')[['LogConfirmedDelta']].count()
], axis=1)
 deltas.mean()
confirmed_deltas.columns = ['avg', 'std', 'cnt']
confirmed_deltas
confirmed_deltas.to_csv('confirmed_deltas.csv' )<feature_engineering>
cols_qualitative, cols_quantitative, cols_flg, cols_unnecessary = [], [], [], []
print('
')
 notebook_begin = time.time()
load_merged_df(cache=True)
 COL_ID = 'ID'
COL_TARGET = 'loan_condition'
print('
')
 print('Adding n_nan')
 df_train['n_nan'] = df_train.isnull().sum(axis=1)
 df_test['n_nan'] = df_test.isnull().sum(axis=1)
 object2date('issue_d')
 object2date('earliest_cr_line')
 extract_recent_data(oldest_year=2014)
 cols_unnecessary.append('issue_d')
 ordinal2int('grade')
 ordinal2int('sub_grade')
 emplength2int()
object2str('title')
 object2str('emp_title')
 preprocess_text('title')
 preprocess_words_of_text('title')
 zipcode2str()
print('
')
 load_cols_by_dtype()
print('
')
 log_transformation('loan_amnt', remove_outlier=False)
 log_transformation('installment', remove_outlier=False)
 log_transformation('annual_inc', remove_outlier=True, add_flg=True)
 log_transformation('revol_bal', remove_outlier=True, add_flg=True)
 log_transformation('tot_coll_amt', remove_outlier=True, add_flg=True)
 log_transformation('tot_cur_bal', remove_outlier=True, add_flg=True)
 log_transformation('EXTER:TotalWages', remove_outlier=False)
 print('
')
 fillna_text('title', add_flg=False)
 fillna_text('emp_title', add_flg=False)
 fillna_numeric('dti', fillna_by=-999)
 fillna_numeric('mths_since_last_delinq', fillna_by=-999)
 fillna_numeric('mths_since_last_record', fillna_by=-999)
 fillna_numeric('revol_util', fillna_by=-999)
 fillna_numeric('mths_since_last_major_derog', fillna_by=-999)
 fillna_numeric('inq_last_6mths', fillna_by='median')
 fillna_numeric('EXTER:TotalWages', fillna_by=-999)
 print('
')
 tfidf_pca_kmeans_cluster('emp_title', n_dim_tfidf=500, n_dim_pca=-1, n_cluster=100)
 tfidf_pca_logreg_prediction('emp_title', n_dim_tfidf=500, n_dim_pca=-1)
 update_cols(add_cols_unnecessary=['emp_title'])
 year2decade('earliest_cr_line', remove_original=True)
 ordinal2int('earliest_cr_line_decade_year')
 record_clustering_kmeans(n_cluster=100)
 print('
')
 cols_qualitative_all = cols_qualitative.copy()
count_encoding(cols_qualitative_all, remove_original=True)
 count_encoding(cols_qualitative_all, remove_original=False)
 target_encoding(cols_qualitative_all, remove_original=True)
 del cols_qualitative_all
cols_quantitative_few_unique = ['acc_now_delinq', 'collections_12_mths_ex_med',
'earliest_cr_line_decade_year', 'emp_length', 'grade',
'inq_last_6mths', 'n_nan']
target_encoding(cols_quantitative_few_unique, remove_original=False)
 for col in cols_quantitative:
if 'TARGET' in col:
df_train[col] = df_train[col].fillna(df_train[col].median())
 df_test[col] = df_test[col].fillna(df_test[col].median())
 print('
')
 generate_interaction_terms(['sub_grade', 'grade', 'TXT:logreg_prediction_on_emp_title', 'loan_amnt', 'dti',
'COUNT:home_ownership', 'tot_cur_bal', 'open_acc', 'installment', 'emp_length'],
n_conbinations=2)
 print(' > {} interaction features added'.format(len([col for col in cols_quantitative if(col not in cols_unnecessary)and('INTRCTN' in col)])))
 generate_difference_terms(['sub_grade', 'grade', 'TXT:logreg_prediction_on_emp_title', 'loan_amnt', 'dti',
'COUNT:home_ownership', 'tot_cur_bal', 'open_acc', 'installment', 'emp_length'])
 print(' > {} difference features added'.format(len([col for col in cols_quantitative if(col not in cols_unnecessary)and('DIFF' in col)])))
 generate_division_terms([['EXTER:State & Local Spending', 'annual_inc'], ['loan_amnt', 'annual_inc'],
['installment', 'annual_inc'], ['tot_coll_amt', 'annual_inc'], ['tot_cur_bal', 'annual_inc'],
['loan_amnt', 'revol_bal'], ['installment', 'revol_bal'], ['tot_coll_amt', 'revol_bal'],
['tot_cur_bal', 'revol_bal'], ['loan_amnt', 'tot_cur_bal'], ['installment', 'tot_cur_bal'],
['tot_coll_amt', 'tot_cur_bal']])
 print(' > {} division features added'.format(len([col for col in cols_quantitative if(col not in cols_unnecessary)and('DIV' in col)])))
 cols_unnecessary_tmp = ['INTRCTN:COUNT:home_ownership(times)emp_length', 'INTRCTN:emp_length(times)grade',
'DIFF:COUNT:home_ownership(minus)emp_length', 'DIFF:dti(minus)loan_amnt',
'INTRCTN:COUNT:home_ownership(times)grade',
'DIFF:TXT:logreg_prediction_on_emp_title(minus)dti', 'DIFF:grade(minus)open_acc',
'DIFF:COUNT:home_ownership(minus)grade', 'DIFF:emp_length(minus)grade',
'DIFF:grade(minus)sub_grade', 'INTRCTN:grade(times)sub_grade']
cols_unnecessary_tmp += [col for col in cols_quantitative if 'DIFF' in col]
update_cols(add_cols_unnecessary=cols_unnecessary_tmp)
 print('
')
 pertition_holdout(rate_holdout=0.2, use_test_like_records=False, use_recent_records=True)
 cross_validation(clf_name='HistGradientBoosting', rate_sampling=1.0, n_folds=5)
 print('
')
 holdout_validation(clf_name='HistGradientBoosting')
 print('
')
 modeling_and_prediction(clf_name='HistGradientBoosting', cross_validation_ensemble=True, n_folds=5)
 print('Appending model predictions to y_preds_list')
 y_preds_list_for_ensemble.append(y_pred_final)
 cvscore_list_for_ensemble.append(score_cv_avr)
 holdoutscore_list_for_ensemble.append(score_holdout)
 print('
')
 print('lgbm_params: {}'.format(lgbm_params))
print('number of features: {}'.format(len(get_cols_to_use())))
 print('Print scores')
 print(' > Avr of CV: {:.5f}'.format(score_cv_avr))
print(' > Holdout : {:.5f}'.format(score_holdout))
print('Calculating total elapsed time')
 elapsed_time = time.time() -notebook_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' > total elapsed time...{}:{}'.format(elapsed_min, str(elapsed_sec ).rjust(2, '0')))
DECAY = 0.93
DECAY ** 7, DECAY ** 14, DECAY ** 21, DECAY ** 28
confirmed_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
confirmed_deltas['DELTA'] = GLOBAL_DELTA
confirmed_deltas.loc[confirmed_deltas.continent=='Africa', 'DELTA'] = 0.14
confirmed_deltas.loc[confirmed_deltas.continent=='Oceania', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Korea South', 'DELTA'] = 0.011
confirmed_deltas.loc[confirmed_deltas.Country_Region=='US', 'DELTA'] = 0.15
confirmed_deltas.loc[confirmed_deltas.Country_Region=='China', 'DELTA'] = 0.01
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Japan', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Norway', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iceland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Austria', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Italy', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Spain', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Portugal', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Israel', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iran', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Germany', 'DELTA'] = 0.07
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Russia', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Brazil', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Turkey', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Philippines', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Location=='France-', 'DELTA'] = 0.1
confirmed_deltas.loc[confirmed_deltas.Location=='United Kingdom-', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
confirmed_deltas.loc[confirmed_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Location=='San Marino-', 'DELTA'] = 0.03
confirmed_deltas.shape, confirmed_deltas.DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].shape, confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA]
confirmed_deltas.describe()<save_to_csv>
cols_qualitative, cols_quantitative, cols_flg, cols_unnecessary = [], [], [], []
print('
')
 notebook_begin = time.time()
load_merged_df(cache=True)
 COL_ID = 'ID'
COL_TARGET = 'loan_condition'
print('
')
 print('Adding n_nan')
 df_train['n_nan'] = df_train.isnull().sum(axis=1)
 df_test['n_nan'] = df_test.isnull().sum(axis=1)
 object2date('issue_d')
 object2date('earliest_cr_line')
 extract_recent_data(oldest_year=2014)
 cols_unnecessary.append('issue_d')
 ordinal2int('grade')
 ordinal2int('sub_grade')
 emplength2int()
object2str('title')
 object2str('emp_title')
 preprocess_text('title')
 preprocess_words_of_text('title')
 zipcode2str()
print('
')
 load_cols_by_dtype()
print('
')
 fillna_text('title', add_flg=False)
 fillna_text('emp_title', add_flg=False)
 fillna_numeric('dti', fillna_by=999.0)
 fillna_numeric('mths_since_last_delinq', fillna_by='minmax', val_min=0, val_max=85, add_flg=True)
 fillna_numeric('mths_since_last_record', fillna_by='random', add_flg=True)
 fillna_numeric('revol_util', fillna_by='minmax', val_min=0, val_max=100, add_flg=True)
 fillna_numeric('mths_since_last_major_derog', fillna_by='minmax', val_min=0, val_max=90, add_flg=True)
 fillna_numeric('inq_last_6mths', fillna_by='median')
 fillna_numeric('EXTER:TotalWages', fillna_by='random', add_flg=True)
 print('
')
 log_transformation('loan_amnt', remove_outlier=False, add_flg=False)
 log_transformation('installment', remove_outlier=False, add_flg=False)
 log_transformation('annual_inc', remove_outlier=True, add_flg=False)
 log_transformation('revol_bal', remove_outlier=True, add_flg=False)
 log_transformation('tot_coll_amt', remove_outlier=True, add_flg=False)
 log_transformation('tot_cur_bal', remove_outlier=True, add_flg=False)
 log_transformation('EXTER:TotalWages', remove_outlier=False)
 print('
')
 tfidf_pca('emp_title', n_dim_tfidf=500, n_dim_pca=10)
 update_cols(add_cols_unnecessary=['title', 'emp_title'])
 year2decade('earliest_cr_line', remove_original=True)
 ordinal2int('earliest_cr_line_decade_year')
 print('
')
 update_cols(add_cols_unnecessary=['zip_code'])
 onehot_encoding(cols_qualitative)
 print('
')
 rank_gauss_transformation(n_split=5)
 print('
')
 print('number of features: {}'.format(len(get_cols_to_use())))
 pertition_holdout(rate_holdout=0.2, use_test_like_records=False, use_recent_records=True)
 cross_validation(clf_name='NeuralNetwork', rate_sampling=1.0)
 gc.collect()
print('
')
 holdout_validation(clf_name='NeuralNetwork')
 gc.collect()
print('
')
 modeling_and_prediction(clf_name='NeuralNetwork', cross_validation_ensemble=True, n_folds=3)
 gc.collect()
print('Appending model predictions to y_preds_list')
 y_preds_list_for_ensemble.append(y_pred_final)
 cvscore_list_for_ensemble.append(score_cv_avr)
 holdoutscore_list_for_ensemble.append(score_holdout)
 print('
')
 print('number of features: {}'.format(len(get_cols_to_use())))
 print('Print scores')
 print(' > Avr of CV: {:.5f}'.format(score_cv_avr))
print(' > Holdout : {:.5f}'.format(score_holdout))
print('Calculating total elapsed time')
 elapsed_time = time.time() -notebook_begin
elapsed_sec = int(elapsed_time%60)
 elapsed_min = int(( elapsed_time-elapsed_sec)/60)
 print(' > total elapsed time...{}:{}'.format(elapsed_min, str(elapsed_sec ).rjust(2, '0')))
daily_log_confirmed = train_clean.pivot('Location', 'Date', 'LogConfirmed' ).reset_index()
daily_log_confirmed = daily_log_confirmed.sort_values(TRAIN_END, ascending=False)
 daily_log_confirmed.to_csv('daily_log_confirmed.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in confirmed_deltas.Location.values:
confirmed_delta = confirmed_deltas.loc[confirmed_deltas.Location == loc, 'DELTA'].values[0]
daily_log_confirmed.loc[daily_log_confirmed.Location == loc, new_day] = daily_log_confirmed.loc[daily_log_confirmed.Location == loc, last_day] + \
confirmed_delta * DECAY ** i<save_to_csv>
model_names = ['LogisticRegression', 'LightGBM(1)', 'LightGBM(2)', 'HistGradientBoosting', 'NeuralNetwork']
df_results = pd.DataFrame({'Model': model_names, 'Avr Score of CV': cvscore_list_for_ensemble, 'Holdout Score': holdoutscore_list_for_ensemble})
 df_results
train_clean['Geo
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_death_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_death_deltas = latest.merge(daily_death_deltas, on='Geo
daily_death_deltas.shape
daily_death_deltas.head()
daily_death_deltas.to_csv('daily_death_deltas.csv', index=False )<feature_engineering>
print('Saving model predictions')
 fname = 'submission_5models_avr_ensemble.csv'
submission = pd.read_csv('{}/sample_submission.csv'.format(DIR), index_col=0)
 submission.loan_condition = np.mean(y_preds_list_for_ensemble, axis=0)
 submission.to_csv(fname)
 print(' > saved: {}'.format(fname))
weights = [3.9, 11.4, 11.0, 11.0, 5.6]
fname = 'submission_5models_weighted_avr_ensemble.csv'
submission = pd.read_csv('{}/sample_submission.csv'.format(DIR), index_col=0)
 submission.loan_condition = np.average(y_preds_list_for_ensemble, weights=weights, axis=0)
 submission.to_csv(fname)
 print(' > saved: {}'.format(fname))
best_3preds_plusNN = [y_preds_list_for_ensemble[1], y_preds_list_for_ensemble[2],
y_preds_list_for_ensemble[3], y_preds_list_for_ensemble[4]]
fname = 'submission_best3models_plusNN_avr_ensemble.csv'
submission = pd.read_csv('{}/sample_submission.csv'.format(DIR), index_col=0)
 submission.loan_condition = np.mean(best_3preds_plusNN, axis=0)
 submission.to_csv(fname)
 print(' > saved: {}'.format(fname))
best_2preds = [y_preds_list_for_ensemble[1], y_preds_list_for_ensemble[2]]
fname = 'submission_2LightGBMs_avr_ensemble.csv'
submission = pd.read_csv('{}/sample_submission.csv'.format(DIR), index_col=0)
 submission.loan_condition = np.mean(best_2preds, axis=0)
 submission.to_csv(fname)
 print(' > saved: {}'.format(fname))