Kaggle competition: Homework for Students
def feature_engineering_oscii() :
def fix_area(x):
try:
x_new = x['Country_Region'] + "/" + x['Province_State']
except:
x_new = x['Country_Region']
return x_new
def fix_area2(x):
try:
x_new = x['Country/Region'] + "/" + x['Province/State']
except:
x_new = x['Country/Region']
return x_new
def encode_label(df, col, freq_limit=0):
df[col][pd.isna(df[col])] = 'nan'
tmp = df[col].value_counts()
cols = tmp.index.values
freq = tmp.values
num_cols =(freq>=freq_limit ).sum()
print("col: {}, num_cat: {}, num_reduced: {}".format(col, len(cols), num_cols))
col_new = '{}_le'.format(col)
 df_new = pd.DataFrame(np.ones(len(df), np.int16)*(num_cols-1), columns=[col_new])
 for i, item in enumerate(cols[:num_cols]):
df_new[col_new][df[col]==item] = i
return df_new
def get_df_le(df, col_index, col_cat):
df_new = df[[col_index]]
for col in col_cat:
df_tmp = encode_label(df, col)
 df_new = pd.concat([df_new, df_tmp], axis=1)
 return df_new
def to_float(x):
x_new = 0
try:
x_new = float(x.replace(",", ""))
except:
x_new = np.nan
return x_new
df_train = pd.read_csv(".. /input/covid19-global-forecasting-week-3/train.csv")
 df_test = pd.read_csv(".. /input/covid19-global-forecasting-week-3/test.csv")
 df_traintest = pd.concat([df_train, df_test])
 print('process_date')
 df_traintest['Date'] = pd.to_datetime(df_traintest['Date'])
 df_traintest['day'] = df_traintest['Date'].apply(lambda x: x.dayofyear ).astype(np.int16)
 day_min = df_traintest['day'].min()
df_traintest['days'] = df_traintest['day'] - day_min
df_traintest.loc[df_traintest['Province_State'].isnull() , 'Province_State'] = 'N/A'
df_traintest['place_id'] = df_traintest['Country_Region'] + '_' + df_traintest['Province_State']
print('add lat and long')
 df_latlong = pd.read_csv(".. /input/smokingstats/df_Latlong.csv")
 df_latlong.loc[df_latlong['Province/State'].isnull() , 'Province_State'] = 'N/A'
df_latlong['place_id'] = df_latlong['Country/Region'] + '_' + df_latlong['Province/State']
df_latlong = df_latlong[df_latlong['place_id'].duplicated() ==False]
df_traintest = pd.merge(df_traintest, df_latlong[['place_id', 'Lat', 'Long']], on='place_id', how='left')
 places = np.sort(df_traintest['place_id'].unique())
 print('calc cases, fatalities per day')
 df_traintest2 = copy.deepcopy(df_traintest)
 df_traintest2['cases/day'] = 0
df_traintest2['fatal/day'] = 0
tmp_list = np.zeros(len(df_traintest2))
for place in places:
tmp = df_traintest2['ConfirmedCases'][df_traintest2['place_id']==place].values
tmp[1:] -= tmp[:-1]
df_traintest2['cases/day'][df_traintest2['place_id']==place] = tmp
tmp = df_traintest2['Fatalities'][df_traintest2['place_id']==place].values
tmp[1:] -= tmp[:-1]
df_traintest2['fatal/day'][df_traintest2['place_id']==place] = tmp
print('do agregation')
 df_traintest3 = []
for place in places[:]:
df_tmp = df_traintest2[df_traintest2['place_id']==place].reset_index(drop=True)
 df_tmp = do_aggregations(df_tmp)
 df_traintest3.append(df_tmp)
 df_traintest3 = pd.concat(df_traintest3 ).reset_index(drop=True)
 print('add smoking')
 df_smoking = pd.read_csv(".. /input/smokingstats/share-of-adults-who-smoke.csv")
 df_smoking_recent = df_smoking.sort_values('Year', ascending=False ).reset_index(drop=True)
 df_smoking_recent = df_smoking_recent[df_smoking_recent['Entity'].duplicated() ==False]
df_smoking_recent['Country_Region'] = df_smoking_recent['Entity']
df_smoking_recent['SmokingRate'] = df_smoking_recent['Smoking prevalence, total(ages 15+ )(% of adults)']
df_traintest4 = pd.merge(df_traintest3, df_smoking_recent[['Country_Region', 'SmokingRate']], on='Country_Region', how='left')
 SmokingRate = df_smoking_recent['SmokingRate'][df_smoking_recent['Entity']=='World'].values[0]
df_traintest4['SmokingRate'][pd.isna(df_traintest4['SmokingRate'])] = SmokingRate
print('add data from World Economic Outlook Database')
 df_weo = pd.read_csv(".. /input/smokingstats/WEO.csv")
 subs = df_weo['Subject Descriptor'].unique() [:-1]
df_weo_agg = df_weo[['Country']][df_weo['Country'].duplicated() ==False].reset_index(drop=True)
 for sub in subs[:]:
df_tmp = df_weo[['Country', '2019']][df_weo['Subject Descriptor']==sub].reset_index(drop=True)
 df_tmp = df_tmp[df_tmp['Country'].duplicated() ==False].reset_index(drop=True)
 df_tmp.columns = ['Country', sub]
df_weo_agg = df_weo_agg.merge(df_tmp, on='Country', how='left')
 df_weo_agg.columns = ["".join(c if c.isalnum() else "_" for c in str(x)) for x in df_weo_agg.columns]
df_weo_agg.columns
df_weo_agg['Country_Region'] = df_weo_agg['Country']
df_traintest5 = pd.merge(df_traintest4, df_weo_agg, on='Country_Region', how='left')
 print('add Life expectancy')
 df_life = pd.read_csv(".. /input/smokingstats/Life expectancy at birth.csv")
 tmp = df_life.iloc[:,1].values.tolist()
df_life = df_life[['Country', '2018']]
def func(x):
x_new = 0
try:
x_new = float(x.replace(",", ""))
except:
x_new = np.nan
return x_new
df_life['2018'] = df_life['2018'].apply(lambda x: func(x))
df_life = df_life[['Country', '2018']]
df_life.columns = ['Country_Region', 'LifeExpectancy']
df_traintest6 = pd.merge(df_traintest5, df_life, on='Country_Region', how='left')
 print("add additional info from countryinfo dataset")
 df_country = pd.read_csv(".. /input/countryinfo/covid19countryinfo.csv")
 df_country['Country_Region'] = df_country['country']
df_country = df_country[df_country['country'].duplicated() ==False]
df_traintest7 = pd.merge(df_traintest6,
df_country.drop(['tests', 'testpop', 'country'], axis=1),
on=['Country_Region',], how='left')
 df_traintest7['id'] = np.arange(len(df_traintest7))
df_le = get_df_le(df_traintest7, 'id', ['Country_Region', 'Province_State'])
 df_traintest8 = pd.merge(df_traintest7, df_le, on='id', how='left')
 print('covert object type to float')
 cols = [
'Gross_domestic_product__constant_prices',
'Gross_domestic_product__current_prices',
'Gross_domestic_product__deflator',
'Gross_domestic_product_per_capita__constant_prices',
'Gross_domestic_product_per_capita__current_prices',
'Output_gap_in_percent_of_potential_GDP',
'Gross_domestic_product_based_on_purchasing_power_parity__PPP__valuation_of_country_GDP',
'Gross_domestic_product_based_on_purchasing_power_parity__PPP__per_capita_GDP',
'Gross_domestic_product_based_on_purchasing_power_parity__PPP__share_of_world_total',
'Implied_PPP_conversion_rate', 'Total_investment',
'Gross_national_savings', 'Inflation__average_consumer_prices',
'Inflation__end_of_period_consumer_prices',
'Six_month_London_interbank_offered_rate__LIBOR_',
'Volume_of_imports_of_goods_and_services',
'Volume_of_Imports_of_goods',
'Volume_of_exports_of_goods_and_services',
'Volume_of_exports_of_goods', 'Unemployment_rate', 'Employment', 'Population',
'General_government_revenue', 'General_government_total_expenditure',
'General_government_net_lending_borrowing', 'General_government_structural_balance',
'General_government_primary_net_lending_borrowing', 'General_government_net_debt',
'General_government_gross_debt', 'Gross_domestic_product_corresponding_to_fiscal_year__current_prices',
'Current_account_balance', 'pop'
]
df_traintest8['cases/day'] = df_traintest8['cases/day'].astype(np.float)
 df_traintest8['fatal/day'] = df_traintest8['fatal/day'].astype(np.float)
 for col in cols:
df_traintest8[col] = df_traintest8[col].apply(lambda x: to_float(x))
return df_traintest8<compute_test_metric>
warnings.filterwarnings('ignore')
 plt.style.use('ggplot')
 %matplotlib inline
def calc_score(y_true, y_pred):
y_true[y_true<0] = 0
score = mean_squared_error(np.log(y_true.clip(0, 1e10)+1), np.log(y_pred[:]+1)) **0.5
return score<init_hyperparams>
df_train = pd.read_csv('.. /input/homework-for-students4plus/train.csv', index_col=0, parse_dates=['issue_d'])
 df_test = pd.read_csv('.. /input/homework-for-students4plus/test.csv', index_col=0, parse_dates=['issue_d'])
 df_zipcode = pd.read_csv('.. /input/homework-for-students4plus/free-zipcode-database.csv', dtype={'Zipcode':np.int64})
 df_gdp = pd.read_csv('.. /input/homework-for-students4plus/US_GDP_by_State.csv')
 df_statelatlong = pd.read_csv('.. /input/homework-for-students4plus/statelatlong.csv')
 df_zipcode = df_zipcode[['Zipcode','Xaxis','Yaxis','Zaxis']]
df_gdp = df_gdp[df_gdp.year>2013]
SEED = 42
params = {'num_leaves': 8,
'min_data_in_leaf': 5,
'objective': 'regression',
'max_depth': 8,
'learning_rate': 0.02,
'boosting': 'gbdt',
'bagging_freq': 5,
'bagging_fraction': 0.8,
'feature_fraction': 0.8201,
'bagging_seed': SEED,
'reg_alpha': 1,
'reg_lambda': 4.9847051755586085,
'random_state': SEED,
'metric': 'mse',
'verbosity': 100,
'min_gain_to_split': 0.02,
'min_child_weight': 5,
'num_threads': 6,
}<define_variables>
df_zipcode = df_zipcode.rename(columns={'Zipcode':'zip_code'})
 df_zipcode['zip_code'] = df_zipcode.zip_code.astype(str ).str[:3].astype(np.int64)
 df_zipcode = df_zipcode[~df_zipcode.duplicated('zip_code')]
df_zipcode.head()
col_target = 'fatal/day'
col_var = [
'Lat', 'Long',
'cases/day_(1-1)',
'cases/day_(1-7)',
'fatal/day_(1-7)',
'fatal/day_(8-14)',
'fatal/day_(15-21)',
'SmokingRate',
'density',
]
col_cat = []
df_train = df_traintest[(pd.isna(df_traintest['ForecastId'])) &(df_traintest['days']<TRAIN_N)]
df_valid = df_traintest[(pd.isna(df_traintest['ForecastId'])) &(df_traintest['days']<TRAIN_N)]
X_train = df_train[col_var]
X_valid = df_valid[col_var]
y_train = np.log(df_train[col_target].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 num_round = 340
model = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150,)
 best_itr = model.best_iteration<define_variables>
df_train['zip_code'] = df_train.zip_code.str[:3].astype(np.int64)
 df_test['zip_code'] = df_test.zip_code.str[:3].astype(np.int64)
 df_train = pd.merge(df_train,df_zipcode,on='zip_code',how='left')
 df_test = pd.merge(df_test,df_zipcode,on='zip_code',how='left')
 df_train.shape
col_target2 = 'cases/day'
col_var2 = [
'Lat', 'Long',
'days_since_10cases',
'cases/day_(1-1)',
'cases/day_(1-7)',
'cases/day_(8-14)',
'cases/day_(15-21)',
]<prepare_x_and_y>
df_statelatlong = df_statelatlong.rename(columns={'State':'addr_state'})
 df_gdp = df_gdp.rename(columns={'State':'City'})
 df_train = pd.merge(df_train,df_statelatlong,on='addr_state',how='left')
 df_test = pd.merge(df_test,df_statelatlong,on='addr_state',how='left' )
df_train = df_traintest[(pd.isna(df_traintest['ForecastId'])) &(df_traintest['days']<TRAIN_N)]
df_valid = df_traintest[(pd.isna(df_traintest['ForecastId'])) &(df_traintest['days']<TRAIN_N)]
X_train = df_train[col_var2]
X_valid = df_valid[col_var2]
y_train = np.log(df_train[col_target2].values.clip(0, 1e10)+1)
 y_valid = np.log(df_valid[col_target2].values.clip(0, 1e10)+1)
 train_data = lgb.Dataset(X_train, label=y_train, categorical_feature=col_cat)
 valid_data = lgb.Dataset(X_valid, label=y_valid, categorical_feature=col_cat)
 model2 = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150,)
 best_itr2 = model2.best_iteration<define_variables>
df_train = df_train[df_train['issue_d'] > '2013-12-31']
places = AREAS.copy()<drop_column>
df_train["earliest_cr_line2"]=pd.to_datetime(df_train["earliest_cr_line"])
 df_test["earliest_cr_line2"]=pd.to_datetime(df_test["earliest_cr_line"])
 df_train["issue_d_unix"] = df_train["issue_d"].view('int64')// 10**9
df_test["issue_d_unix"] = df_test["issue_d"].view('int64')// 10**9
df_train["earliest_cr_line_unix"] = df_train["earliest_cr_line2"].view('int64')// 10**9
df_test["earliest_cr_line_unix"] = df_test["earliest_cr_line2"].view('int64')// 10**9
df_train["period"]=df_train["issue_d_unix"]-df_train["earliest_cr_line_unix"]
df_test["period"]=df_test["issue_d_unix"]-df_test["earliest_cr_line_unix"]
df_train["period"]=df_train["period"].fillna(0)
 df_test["period"]=df_test["period"].fillna(0)
 df_train['loan_amnt__installment']=round(df_train['loan_amnt']/df_train['installment'],5)
 df_test['loan_amnt__installment']=round(df_test['loan_amnt']/df_test['installment'],5)
 df_train['loan_amnt__annual_inc']=round(df_train['loan_amnt']/df_train['annual_inc'],5)
 df_test['loan_amnt__annual_inc']=round(df_test['loan_amnt']/df_test['annual_inc'],5)
 df_train['revol_bal__revol_util']=round(df_train['revol_bal']/df_train['revol_util'],5)
 df_test['revol_bal__revol_util']=round(df_test['revol_bal']/df_test['revol_util'],5)
 df_train['revol_bal__total_acc']=round(df_train['revol_bal']/df_train['total_acc'],5)
 df_test['revol_bal__total_acc']=round(df_test['revol_bal']/df_test['total_acc'],5)
 df_train['loan_amnt__open_acc']=round(df_train['loan_amnt']/df_train['open_acc'],5)
 df_test['loan_amnt__open_acc']=round(df_test['loan_amnt']/df_test['open_acc'],5 )
df_tmp = df_traintest[(( df_traintest['days']<TRAIN_N)&(pd.isna(df_traintest['ForecastId'])))|(( TRAIN_N<=df_traintest['days'])&(pd.isna(df_traintest['ForecastId'])==False)) ].reset_index(drop=True)
 df_tmp = df_tmp.drop([
'cases/day_(1-1)', 'cases/day_(1-7)', 'cases/day_(8-14)', 'cases/day_(15-21)',
'fatal/day_(1-1)', 'fatal/day_(1-7)', 'fatal/day_(8-14)', 'fatal/day_(15-21)',
'days_since_1cases', 'days_since_10cases', 'days_since_100cases',
'days_since_1fatal', 'days_since_10fatal', 'days_since_100fatal',
], axis=1)
 df_traintest9 = []
for i, place in tqdm(enumerate(places[:])) :
df_tmp2 = df_tmp[df_tmp['place_id']==place].reset_index(drop=True)
 df_tmp2 = do_aggregations(df_tmp2)
 df_traintest9.append(df_tmp2)
 df_traintest9 = pd.concat(df_traintest9 ).reset_index(drop=True)
<filter>
y_train = df_train.loan_condition
X_train = df_train.drop(['loan_condition'], axis=1)
 X_test = df_test
df_preds = []
for i, place in tqdm(enumerate(places[:])) :
df_interest = copy.deepcopy(df_traintest9[df_traintest9['place_id']==place].reset_index(drop=True))
df_interest['cases/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
df_interest['fatal/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
len_known =(df_interest['days']<TRAIN_N ).sum()
len_unknown = 30
for j in range(len_unknown):
X_valid = df_interest[col_var].iloc[j+len_known]
X_valid2 = df_interest[col_var2].iloc[j+len_known]
pred_f = model.predict(X_valid)
 pred_c = model2.predict(X_valid2)
 pred_c =(np.exp(pred_c)-1 ).clip(0, 1e10)
 pred_f =(np.exp(pred_f)-1 ).clip(0, 1e10)
 df_interest['fatal/day'][j+len_known] = pred_f
df_interest['cases/day'][j+len_known] = pred_c
df_interest['Fatalities'][j+len_known] = df_interest['Fatalities'][j+len_known-1] + pred_f
df_interest['ConfirmedCases'][j+len_known] = df_interest['ConfirmedCases'][j+len_known-1] + pred_c
df_interest = df_interest.drop([
'cases/day_(1-1)', 'cases/day_(1-7)', 'cases/day_(8-14)', 'cases/day_(15-21)',
'fatal/day_(1-1)', 'fatal/day_(1-7)', 'fatal/day_(8-14)', 'fatal/day_(15-21)',
'days_since_1cases', 'days_since_10cases', 'days_since_100cases',
'days_since_1fatal', 'days_since_10fatal', 'days_since_100fatal',
], axis=1)
 df_interest = do_aggregations(df_interest)
 if(i+1)%10==0:
print("{:3d}/{} {}, len known: {}, len unknown: {}".format(i+1, len(places), place, len_known, len_unknown), df_interest.shape)
 df_interest['fatal_pred'] = np.cumsum(df_interest['fatal/day'].values)
 df_interest['cases_pred'] = np.cumsum(df_interest['cases/day'].values)
 df_preds.append(df_interest)
 df_preds = pd.concat(df_preds )<prepare_output>
X_train.drop('issue_d', axis=1, inplace=True)
 X_test.drop('issue_d', axis=1, inplace=True)
 X_train.drop('earliest_cr_line2', axis=1, inplace=True)
 X_test.drop('earliest_cr_line2', axis=1, inplace=True)
preds_f_oscii = np.log1p(p_f_oscii.values[:].copy())
 preds_c_oscii = np.log1p(p_c_oscii.values[:].copy())
 preds_f_oscii.shape, preds_c_oscii.shape<compute_test_metric>
cols1 = ['loan_amnt','installment','installment','loan_amnt','loan_amnt']
cols2 = ['revol_bal','revol_bal','revol_util','open_acc','revol_util']
for col1,col2 in zip(cols1,cols2):
col12m = col1+'&'+col2
X_train[col12m]=X_train[col1]*X_train[col2]
X_test[col12m]=X_test[col1]*X_test[col2]
if False:
val_len = train_p_c.values.shape[1] - TRAIN_N
m1s = []
m2s = []
for i in range(val_len):
d = i + TRAIN_N
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, d]), preds_c_oscii[:, d-START_PUBLIC]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, d]), preds_f_oscii[:, d-START_PUBLIC]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 m1s += [m1]
m2s += [m2]
print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_c_oscii[:, TRAIN_N-START_PUBLIC:TRAIN_N-START_PUBLIC+val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_f_oscii[:, TRAIN_N-START_PUBLIC:TRAIN_N-START_PUBLIC+val_len].flatten()))
print(f"{(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]" )<define_search_space>
cats = []
for col in X_train.columns:
if X_train[col].dtype == 'object':
cats.append(col)
 print(col, X_train[col].nunique() )
preds_c_blend = np.log1p(np.average([np.expm1(preds_c_oscii[:,64:107]),np.expm1(preds_c_cpmp[:]),np.expm1(p_c_beluga[:,64:107]),np.expm1(preds_c[:,64:107])],axis=0, weights=[8,1,1,8]))
preds_f_blend = np.log1p(np.average([np.expm1(preds_f_oscii[:,64:107]),np.expm1(preds_f_cpmp[:]),np.expm1(p_f_beluga[:,64:107]),np.expm1(preds_f[:,64:107])],axis=0, weights=[8,1,1,8]))<compute_test_metric>
X_train.fillna('
X_test.fillna('
val_len = 13
m1s = []
m2s = []
for i in range(val_len):
d = i + 64
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, d]), preds_c_blend[:, i]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, d]), preds_f_blend[:, i]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 m1s += [m1]
m2s += [m2]
print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c_raw.values[:, 64:64+val_len] ).flatten() , preds_c_blend[:, :val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f_raw.values[:, 64:64+val_len] ).flatten() , preds_f_blend[:, :val_len].flatten()))
print(f"PUBLIC LB {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]" )<define_variables>
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 for col in cats:
summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 skf = StratifiedKFold(n_splits=5, random_state=99, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train
EU_COUNTRIES = ['Austria', 'Italy', 'Belgium', 'Latvia', 'Bulgaria', 'Lithuania', 'Croatia', 'Luxembourg', 'Cyprus', 'Malta', 'Czechia',
'Netherlands', 'Denmark', 'Poland', 'Estonia', 'Portugal', 'Finland', 'Romania', 'France', 'Slovakia', 'Germany', 'Slovenia',
'Greece', 'Spain', 'Hungary', 'Sweden', 'Ireland']
EUROPE_OTHER = ['Albania', 'Andorra', 'Bosnia and Herzegovina', 'Liechtenstein', 'Monaco', 'Montenegro', 'North Macedonia',
'Norway', 'San Marino', 'Serbia', 'Switzerland', 'Turkey', 'United Kingdom']
AFRICA = ['Algeria', 'Burkina Faso', 'Cameroon', 'Congo(Kinshasa)', "Cote d'Ivoire", 'Egypt', 'Ghana', 'Kenya', 'Madagascar',
'Morocco', 'Nigeria', 'Rwanda', 'Senegal', 'South Africa', 'Togo', 'Tunisia', 'Uganda', 'Zambia']
NORTH_AMERICA = ['US', 'Canada', 'Mexico']
SOUTH_AMERICA = ['Argentina', 'Bolivia', 'Brazil', 'Chile', 'Colombia', 'Ecuador', 'Paraguay', 'Peru', 'Uruguay', 'Venezuela']
MIDDLE_EAST = ['Afghanistan', 'Bahrain', 'Iran', 'Iraq', 'Israel', 'Jordan', 'Kuwait', 'Lebanon', 'Oman', 'Qatar', 'Saudi Arabia', 'United Arab Emirates']
ASIA = ['Bangladesh', 'Brunei', 'Cambodia', 'India', 'Indonesia', 'Japan', 'Kazakhstan', 'Korea, South', 'Kyrgyzstan', 'Malaysia',
'Pakistan', 'Singapore', 'Sri Lanka', 'Taiwan*', 'Thailand', 'Uzbekistan', 'Vietnam']<data_type_conversions>
X_train['grade_subgrade'] = X_train['grade'] * X_train['sub_grade']
X_test['grade_subgrade'] = X_test['grade'] * X_test['sub_grade']
cols1 = ['grade','grade','grade','grade','grade','grade']
cols3 = ['sub_grade','sub_grade','sub_grade','sub_grade','sub_grade','sub_grade']
cols2 = ['dti','emp_title','loan_amnt','annual_inc','tot_cur_bal','home_ownership']
cols1.extend(cols3)
 cols2.extend(cols2)
 add_cols = []
for col1,col2 in zip(cols1,cols2):
col12m = col1+'&'+col2
X_train[col12m]=X_train[col1]*X_train[col2]
X_test[col12m]=X_test[col1]*X_test[col2]
add_cols.append(col12m )
non_china_mask = np.array(['China' not in a for a in AREAS] ).astype(bool)
 non_china_mask.shape 
<prepare_output>
X_train.fillna(X_train.median() , inplace=True)
 X_test.fillna(X_test.median() , inplace=True )
preds_c2 = preds_c.copy()
preds_f2 = preds_f.copy()
preds_c2[non_china_mask,64:107] = preds_c_blend[non_china_mask]
preds_f2[non_china_mask,64:107] = preds_f_blend[non_china_mask]<merge>
X_train_, X_val, y_train_, y_val= train_test_split(X_train, y_train, test_size=0.05, random_state=71 )
temp = pd.DataFrame(np.clip(np.exp(preds_c2)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="ConfirmedCases")
 test = test_orig.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 temp = pd.DataFrame(np.clip(np.exp(preds_f2)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="Fatalities")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 test.head()<save_to_csv>
from hyperopt import fmin, tpe, hp, rand, Trials
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score
from lightgbm import LGBMClassifier
test.to_csv("submission.csv", index=False, columns=["ForecastId", "ConfirmedCases", "Fatalities"] )<count_unique_values>
def objective(space):
scores = []
skf = StratifiedKFold(n_splits=5, random_state=99, shuffle=True)
 for i,(train_ix, test_ix)in enumerate(tqdm(skf.split(X_train, y_train))):
X_train_, y_train_ = X_train.values[train_ix], y_train.values[train_ix]
X_val, y_val = X_train.values[test_ix], y_train.values[test_ix]
clf = LGBMClassifier(n_estimators=9999, **space)
 clf.fit(X_train_, y_train_, early_stopping_rounds=20, eval_metric='auc', eval_set=[(X_val, y_val)])
 y_pred = clf.predict_proba(X_val)[:,1]
score = roc_auc_score(y_val, y_pred)
 scores.append(score)
 scores = np.array(scores)
 print(scores.mean())
 return -scores.mean()
test.days.nunique()<sort_values>
space ={
'max_depth': hp.choice('max_depth', np.arange(1, 50, dtype=int)) ,
'subsample': hp.uniform('subsample', 0.8, 1),
'num_leaves':hp.choice('num_leaves', np.arange(100, 1000)) ,
'learning_rate' : hp.quniform('learning_rate', 0.025, 0.5, 0.025),
'colsample_bytree' : hp.quniform('colsample_bytree', 0.5, 1, 0.05)
 }
for i, rec in test.groupby('Area' ).last().sort_values("ConfirmedCases", ascending=False ).iterrows() :
print(f"{rec['ConfirmedCases']:10.1f} {rec['Fatalities']:10.1f} {rec['Country_Region']}, {rec['Province_State']}")
<set_options>
trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=20,
trials=trials,
rstate=np.random.RandomState(99)
 )
%matplotlib inline 
<load_from_csv>
train=pd.read_csv("/kaggle/input/titanic/train.csv")
 test=pd.read_csv('/kaggle/input/titanic/test.csv')
 train.head(10 )<count_missing_values>
clf_best = LGBMClassifier(**best )
train.isnull().sum()<count_missing_values>
%%time
scores = []
y_pred_test = np.zeros(len(X_test))
skf = StratifiedKFold(n_splits=10, random_state=99, shuffle=True)
 for i,(train_ix, test_ix)in enumerate(tqdm(skf.split(X_train, y_train))):
X_train_, y_train_ = X_train.iloc[train_ix], y_train.iloc[train_ix]
X_val, y_val = X_train.iloc[test_ix], y_train.iloc[test_ix]
clf_best.fit(X_train_, y_train_)
 y_pred = clf_best.predict_proba(X_val)[:,1]
scores.append(roc_auc_score(y_val, y_pred))
y_pred_test += clf_best.predict_proba(X_test)[:,1]
scores = np.array(scores)
 print('Ave.CV score is %f' % scores.mean())
 y_pred_test /= 10
test.isnull().sum()<concatenate>
submission = pd.read_csv('.. /input/homework-for-students4plus/sample_submission.csv', index_col=0)
 submission.loan_condition = y_pred_test
submission.to_csv('submission.csv' )
<feature_engineering><EOS> 
LGBMClassifier(**best )