Kaggle competition: Exam for Students20200129
<SOS> metric: rmsle Kaggle data source: exam-for-students20200129<save_to_csv>
%matplotlib inline
font = {"family":"HGMaruGothicMPRO"}
matplotlib.rc("font",**font)
submission = pd.read_csv('.. /input/homework-for-students4plus/sample_submission.csv', index_col=0)
 submission.loan_condition = y_pred
submission.to_csv('submission.csv' )<set_options>
train = pd.read_csv(".. /input/exam-for-students20200129/train.csv")
 test = pd.read_csv(".. /input/exam-for-students20200129/test.csv")
 country = pd.read_csv(".. /input/exam-for-students20200129/country_info.csv" )
warnings.filterwarnings('ignore')
 plt.style.use('ggplot')
 %matplotlib inline 
<load_from_csv>
train["ConvertedSalary"] = np.log1p(train[["ConvertedSalary"]] )
df_train = pd.read_csv('.. /input/homework-for-students4plus/train.csv', index_col=0, parse_dates=['issue_d'])
 df_test = pd.read_csv('.. /input/homework-for-students4plus/test.csv', index_col=0, parse_dates=['issue_d'])
 df_zipcode = pd.read_csv('.. /input/homework-for-students4plus/free-zipcode-database.csv', dtype={'Zipcode':np.int64})
 df_gdp = pd.read_csv('.. /input/homework-for-students4plus/US_GDP_by_State.csv')
 df_statelatlong = pd.read_csv('.. /input/homework-for-students4plus/statelatlong.csv')
 df_zipcode = df_zipcode[['Zipcode','Xaxis','Yaxis','Zaxis']]
df_gdp = df_gdp[df_gdp.year>2013]<data_type_conversions>
train["hantei"] = "train"
test["hantei"] = "test"
full_data= pd.concat([train, test] )
df_zipcode = df_zipcode.rename(columns={'Zipcode':'zip_code'})
 df_zipcode['zip_code'] = df_zipcode.zip_code.astype(str ).str[:3].astype(np.int64)
 df_zipcode = df_zipcode[~df_zipcode.duplicated('zip_code')]
df_zipcode.head()<data_type_conversions>
full_data = full_data.merge(country, on='Country', how='left' ).set_index(full_data.index )
df_train['zip_code'] = df_train.zip_code.str[:3].astype(np.int64)
 df_test['zip_code'] = df_test.zip_code.str[:3].astype(np.int64)
 df_train = pd.merge(df_train,df_zipcode,on='zip_code',how='left')
 df_test = pd.merge(df_test,df_zipcode,on='zip_code',how='left')
 df_train.shape<merge>
print(train.nunique() )
df_statelatlong = df_statelatlong.rename(columns={'State':'addr_state'})
 df_gdp = df_gdp.rename(columns={'State':'City'})
 df_train = pd.merge(df_train,df_statelatlong,on='addr_state',how='left')
 df_test = pd.merge(df_test,df_statelatlong,on='addr_state',how='left' )<filter>
for i in retsumei:
print(i,train[i].nunique() )
df_train = df_train[df_train['issue_d'] > '2013-12-31']<data_type_conversions>
full_retsumei = []
full_retsumei = full_data.columns
for j in full_retsumei:
print(j,full_data[j].nunique() ,full_data[j].dtype )
df_train["earliest_cr_line2"]=pd.to_datetime(df_train["earliest_cr_line"])
 df_test["earliest_cr_line2"]=pd.to_datetime(df_test["earliest_cr_line"])
 df_train["issue_d_unix"] = df_train["issue_d"].view('int64')// 10**9
df_test["issue_d_unix"] = df_test["issue_d"].view('int64')// 10**9
df_train["earliest_cr_line_unix"] = df_train["earliest_cr_line2"].view('int64')// 10**9
df_test["earliest_cr_line_unix"] = df_test["earliest_cr_line2"].view('int64')// 10**9
df_train["period"]=df_train["issue_d_unix"]-df_train["earliest_cr_line_unix"]
df_test["period"]=df_test["issue_d_unix"]-df_test["earliest_cr_line_unix"]
df_train["period"]=df_train["period"].fillna(0)
 df_test["period"]=df_test["period"].fillna(0)
 df_train['loan_amnt__installment']=round(df_train['loan_amnt']/df_train['installment'],5)
 df_test['loan_amnt__installment']=round(df_test['loan_amnt']/df_test['installment'],5)
 df_train['loan_amnt__annual_inc']=round(df_train['loan_amnt']/df_train['annual_inc'],5)
 df_test['loan_amnt__annual_inc']=round(df_test['loan_amnt']/df_test['annual_inc'],5)
 df_train['revol_bal__revol_util']=round(df_train['revol_bal']/df_train['revol_util'],5)
 df_test['revol_bal__revol_util']=round(df_test['revol_bal']/df_test['revol_util'],5)
 df_train['revol_bal__total_acc']=round(df_train['revol_bal']/df_train['total_acc'],5)
 df_test['revol_bal__total_acc']=round(df_test['revol_bal']/df_test['total_acc'],5)
 df_train['loan_amnt__open_acc']=round(df_train['loan_amnt']/df_train['open_acc'],5)
 df_test['loan_amnt__open_acc']=round(df_test['loan_amnt']/df_test['open_acc'],5 )<prepare_x_and_y>
full_retsumei = []
full_retsumei = full_data.columns
for j in full_retsumei:
if full_data[j].nunique() >50:
if full_data[j].dtype =="object":
print(j,full_data[j].nunique() ,full_data[j].dtype )
y_train = df_train.loan_condition
X_train = df_train.drop(['loan_condition'], axis=1)
 X_test = df_test<drop_column>
X_train.drop('issue_d', axis=1, inplace=True)
 X_test.drop('issue_d', axis=1, inplace=True)
 X_train.drop('earliest_cr_line2', axis=1, inplace=True)
 X_test.drop('earliest_cr_line2', axis=1, inplace=True)
<feature_engineering>
henkan = ["Pop.Density(per sq.mi.) ","Coastline(coast/area ratio)","Net migration","Infant mortality(per 1000 births)"
,"Literacy(%)","Phones(per 1000)","Arable(%)","Crops(%)","Other(%)"
,"Birthrate","Deathrate","Agriculture","Industry","Service"]
for k in henkan:
print(k)
 full_data[k] = full_data[k].str.replace(',', '.' )
cols1 = ['loan_amnt','installment','installment','loan_amnt','loan_amnt']
cols2 = ['revol_bal','revol_bal','revol_util','open_acc','revol_util']
for col1,col2 in zip(cols1,cols2):
col12m = col1+'&'+col2
X_train[col12m]=X_train[col1]*X_train[col2]
X_test[col12m]=X_test[col1]*X_test[col2]<count_unique_values>
print(full_data['Service'].astype("float64"))
cats = []
for col in X_train.columns:
if X_train[col].dtype == 'object':
cats.append(col)
 print(col, X_train[col].nunique() )<prepare_x_and_y>
for l in henkan:
print(full_data[l].astype("float64"))
X_train.fillna('
X_test.fillna('<categorify>
for l in henkan:
full_data[l] = full_data[l].astype("float64")
target = 'loan_condition'
X_temp = pd.concat([X_train, y_train], axis=1)
 for col in cats:
summary = X_temp.groupby([col])[target].mean()
X_test[col] = X_test[col].map(summary)
 skf = StratifiedKFold(n_splits=5, random_state=99, shuffle=True)
 enc_train = Series(np.zeros(len(X_train)) , index=X_train.index)
 for i,(train_ix, val_ix)in enumerate(( skf.split(X_train, y_train))):
X_train_, _ = X_temp.iloc[train_ix], y_train.iloc[train_ix]
X_val, _ = X_temp.iloc[val_ix], y_train.iloc[val_ix]
summary = X_train_.groupby([col])[target].mean()
enc_train.iloc[val_ix] = X_val[col].map(summary)
 X_train[col] = enc_train<feature_engineering>
X_train['grade_subgrade'] = X_train['grade'] * X_train['sub_grade']
X_test['grade_subgrade'] = X_test['grade'] * X_test['sub_grade']
cols1 = ['grade','grade','grade','grade','grade','grade']
cols3 = ['sub_grade','sub_grade','sub_grade','sub_grade','sub_grade','sub_grade']
cols2 = ['dti','emp_title','loan_amnt','annual_inc','tot_cur_bal','home_ownership']
cols1.extend(cols3)
 cols2.extend(cols2)
 add_cols = []
for col1,col2 in zip(cols1,cols2):
col12m = col1+'&'+col2
X_train[col12m]=X_train[col1]*X_train[col2]
X_test[col12m]=X_test[col1]*X_test[col2]
add_cols.append(col12m )<prepare_x_and_y>
for j in full_retsumei:
if full_data[j].nunique() >50:
if full_data[j].dtype =="object":
print(j,full_data[j].nunique() ,full_data[j].dtype )
X_train.fillna(X_train.median() , inplace=True)
 X_test.fillna(X_test.median() , inplace=True )<split>
for j in full_retsumei:
if full_data[j].nunique() <30 and full_data[j].nunique() > 2:
if full_data[j].dtype =="object":
print(j,full_data[j].nunique() ,full_data[j].dtype )
X_train_, X_val, y_train_, y_val= train_test_split(X_train, y_train, test_size=0.05, random_state=71 )<import_modules>
from hyperopt import fmin, tpe, hp, rand, Trials
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score
from lightgbm import LGBMClassifier<train_model>
def objective(space):
scores = []
skf = StratifiedKFold(n_splits=5, random_state=99, shuffle=True)
 for i,(train_ix, test_ix)in enumerate(tqdm(skf.split(X_train, y_train))):
X_train_, y_train_ = X_train.values[train_ix], y_train.values[train_ix]
X_val, y_val = X_train.values[test_ix], y_train.values[test_ix]
clf = LGBMClassifier(n_estimators=9999, **space)
 clf.fit(X_train_, y_train_, early_stopping_rounds=20, eval_metric='auc', eval_set=[(X_val, y_val)])
 y_pred = clf.predict_proba(X_val)[:,1]
score = roc_auc_score(y_val, y_pred)
 scores.append(score)
 scores = np.array(scores)
 print(scores.mean())
 return -scores.mean()<init_hyperparams>
full_data["Age"].value_counts()
space ={
'max_depth': hp.choice('max_depth', np.arange(1, 50, dtype=int)) ,
'subsample': hp.uniform('subsample', 0.8, 1),
'num_leaves':hp.choice('num_leaves', np.arange(100, 1000)) ,
'learning_rate' : hp.quniform('learning_rate', 0.025, 0.5, 0.025),
'colsample_bytree' : hp.quniform('colsample_bytree', 0.5, 1, 0.05)
 }<choose_model_class>
full_data["Age"] = full_data["Age"].str.replace('Under 18 years old', '1')
 full_data["Age"] = full_data["Age"].str.replace('18 - 24 years old', '2')
 full_data["Age"] = full_data["Age"].str.replace('25 - 34 years old', '3')
 full_data["Age"] = full_data["Age"].str.replace('35 - 44 years old', '4')
 full_data["Age"] = full_data["Age"].str.replace('45 - 54 years old', '5')
 full_data["Age"] = full_data["Age"].str.replace('55 - 64 years old', '6')
 full_data["Age"] = full_data["Age"].str.replace('65 years or older', '7' )
trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=20,
trials=trials,
rstate=np.random.RandomState(99)
 )<define_search_space>
full_data['Age'].astype("float64" )
<choose_model_class>
full_data["Age"].value_counts()
clf_best = LGBMClassifier(**best )<find_best_model_class>
full_retsumei = []
full_retsumei = full_data.columns
ordinal = []
for j in full_retsumei:
if full_data[j].dtype =="object":
print(j,full_data[j].nunique() ,full_data[j].dtype)
 ordinal.append(j )
%%time
scores = []
y_pred_test = np.zeros(len(X_test))
skf = StratifiedKFold(n_splits=10, random_state=99, shuffle=True)
 for i,(train_ix, test_ix)in enumerate(tqdm(skf.split(X_train, y_train))):
X_train_, y_train_ = X_train.iloc[train_ix], y_train.iloc[train_ix]
X_val, y_val = X_train.iloc[test_ix], y_train.iloc[test_ix]
clf_best.fit(X_train_, y_train_)
 y_pred = clf_best.predict_proba(X_val)[:,1]
scores.append(roc_auc_score(y_val, y_pred))
y_pred_test += clf_best.predict_proba(X_test)[:,1]
scores = np.array(scores)
 print('Ave.CV score is %f' % scores.mean())
 y_pred_test /= 10<save_to_csv>
oe = ce.OrdinalEncoder(cols=ordinal, return_df=False)
 full_data[ordinal] = oe.fit_transform(full_data[ordinal] )
submission = pd.read_csv('.. /input/homework-for-students4plus/sample_submission.csv', index_col=0)
 submission.loan_condition = y_pred_test
submission.to_csv('submission.csv' )<choose_model_class>
full_data.fillna(-1, inplace=True )
LGBMClassifier(**best )<set_options>
%matplotlib inline<set_options>
full_data["hantei"].value_counts()
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<load_pretrained>
len(full_data[full_data['hantei'] == 1] )
Path('models' ).mkdir(exist_ok=True)
 !cp '.. /input/resnet34/resnet34.pth' 'models/'
def load_url(*args, **kwargs):
model_dir = Path('models')
 filename = 'resnet34.pth'
if not(model_dir/filename ).is_file() : raise FileNotFoundError
return torch.load(model_dir/filename)
 model_zoo.load_url = load_url<load_from_csv>
X_train = full_data[full_data['hantei'] == 1]
X_test = full_data[full_data['hantei'] == 2]
X_train.drop(['hantei'], axis=1, inplace=True)
 X_test.drop(['hantei'], axis=1, inplace=True )
data_path = Path(".. /input/aerial-cactus-identification")
 train_df = pd.read_csv(data_path/'train.csv')
 test_df = pd.read_csv(data_path/'sample_submission.csv' )<concatenate>
y_train = X_train["ConvertedSalary"]
X_train2 = X_train.drop(["ConvertedSalary"], axis=1)
 X_test = X_test.drop(["ConvertedSalary"], axis=1)
 X_train2.sort_index(axis=1, ascending=False)
 X_test.sort_index(axis=1, ascending=False )
df1 = train_df[train_df.has_cactus==0].copy()
df2 = df1.copy()
train_df = train_df.append([df1, df2], ignore_index=True )<feature_engineering>
X_train1,X_test1,y_train,y_test = train_test_split(X_train2,y_train )
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=20, max_lighting=0.3, max_warp=0.2, max_zoom=1.2 )<categorify>
model = lgb.LGBMRegressor()
model.fit(X_train1, y_train )
test_images = ImageList.from_df(test_df, path=data_path/'test', folder='test')
 src =(ImageList.from_df(train_df, path=data_path/'train', folder='train')
.split_by_rand_pct(0.2)
.label_from_df() 
.add_test(test_images))<normalization>
pred = model.predict(X_test1 )
data =(src.transform(tfms,
size=128,
resize_method=ResizeMethod.PAD,
padding_mode='reflection')
.databunch(bs=256)
.normalize(imagenet_stats))<find_best_params>
pred1 = np.expm1(pred )
data.classes, data.c<define_variables>
pred = model.predict(X_test )
data.show_batch(rows=3, figsize=(9,9))<choose_model_class>
pred1 = np.expm1(pred )
<train_model><EOS> 
submission = pd.read_csv('.. /input/exam-for-students20200129/sample_submission.csv', index_col=0)
 submission['ConvertedSalary'] = pred1
submission.to_csv('submission.csv' )