Kaggle competition: Forest Cover Type Prediction
<SOS> metric: CategorizationAccuracy Kaggle data source: forest-cover-type-prediction<load_from_csv>
pd.set_option('display.max_columns', 100)
 pd.options.mode.chained_assignment = None
result_1=pd.read_csv(".. /input/lgb-argu-0407/lgb_submission.csv")
 result_2=pd.read_csv(".. /input/lgb-without-argu-0405/lgb_submission.csv")
 result_3=pd.read_csv(".. /input/no-training-challenge/submission.csv")
 result_4=pd.read_csv(".. /input/nffm-0405-xdeepfm/nffm_submission.csv")
 result_5=pd.read_csv(".. /input/lgb-2-leaves-augment/lgb_submission.csv")
 result_6=pd.read_csv(".. /input/lgb-2-leaves-augment-mine/lgb_submission.csv")
 result_7=pd.read_csv(".. /input/lgb-noaug-pdf200/lgb_submission.csv" )<feature_engineering>
train_path = '.. /input/forest-cover-type-prediction/train.csv'
test_path = '.. /input/forest-cover-type-prediction/test.csv'
submit_path = '.. /input/forest-cover-type-prediction/sampleSubmission.csv'
result_1['index_1']=pd.DataFrame(rankdata(result_1['target'].tolist())) /len(result_1)
 result_2['index_2']=pd.DataFrame(rankdata(result_2['target'].tolist())) /len(result_2)
 result_3['index_3']=pd.DataFrame(rankdata(result_3['target'].tolist())) /len(result_3)
 result_4['index_4']=pd.DataFrame(rankdata(result_4['target'].tolist())) /len(result_4)
 result_5['index_5']=pd.DataFrame(rankdata(result_5['target'].tolist())) /len(result_5)
 result_6['index_6']=pd.DataFrame(rankdata(result_6['target'].tolist())) /len(result_6)
 result_7['index_7']=pd.DataFrame(rankdata(result_7['target'].tolist())) /len(result_7 )<load_pretrained>
dtrain = pd.read_csv(train_path, index_col=0)
 dtest = pd.read_csv(test_path, index_col=0 )
pri_lb=np.load(".. /input/list-of-fake-samples-and-public-private-lb-e2795a/private_LB.npy")
 pub_lb=np.load(".. /input/list-of-fake-samples-and-public-private-lb-e2795a/public_LB.npy")
 df_test_new=result_1.iloc[list(pri_lb)+list(pub_lb),:][['ID_code']]
df_test_new.head()<feature_engineering>
dtrain['Cover_Type'].value_counts()
df_test_new['real']=1
df_test_new.head()<merge>
grouped_dataframe = dtrain.groupby(['Cover_Type'])
 label_dict = {1 : 'Spruce/Fir', 2 : 'Lodgepole Pine', 3 : 'Ponderosa Pine', 4 : 'Cottonwood/Willow', 5 :
'Aspen', 6 : 'Douglas-fir', 7 : 'Krummholz'}
result_avg=pd.merge(result_1,result_2,on='ID_code')
 result_avg=pd.merge(result_avg,result_3,on='ID_code')
 result_avg=pd.merge(result_avg,result_4,on='ID_code')
 result_avg=pd.merge(result_avg,result_5,on='ID_code')
 result_avg=pd.merge(result_avg,result_6,on='ID_code')
 result_avg=pd.merge(result_avg,result_7,on='ID_code' )<merge>
dtrain.drop(['Soil_Type7', 'Soil_Type15'], axis = 1, inplace = True)
 soil_columns = dtrain.columns[dtrain.columns.str.contains('Soil')].to_list()
result_avg=result_avg.merge(df_test_new,on='ID_code',how='left')
 result_avg.head()<count_values>
label = dtrain['Cover_Type']
dtrain.drop(['Cover_Type'], axis = 1, inplace=True )
result_avg.fillna(0,inplace=True)
 result_avg.real.value_counts()<feature_engineering>
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from sklearn.metrics import accuracy_score, f1_score, classification_report,confusion_matrix
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
index_list=[]
for i in result_avg.columns:
if 'index_' in i:
index_list.append(i)
 result_avg['target']=0.0
for i in index_list:
result_avg['target']+=result_avg[i]/len(index_list )<save_to_csv>
x_train, x_test, y_train, y_test = train_test_split(dtrain, label, test_size =.3)
 dirty_clf = RandomForestClassifier()
dirty_clf.fit(x_train, y_train)
 print(dirty_clf.score(x_test, y_test))
imp_feat = pd.DataFrame(index= dtrain.columns.to_list() , data= dirty_clf.feature_importances_)
 imp_feat.rename(columns={0 : 'Importance'}, inplace=True)
 imp_feat.sort_values(by='Importance', axis =0, ascending=False)[:15]
result=result_avg[['ID_code','target']]
ymd=time.strftime("%Y%m%d")
 hms=time.strftime("%H%M%S")
 name="blend_all_{0}_{1}.csv".format(ymd,hms)
 result.to_csv(name,encoding='utf-8',index=None )<import_modules>
baseline_features = ['Elevation', 'Horizontal_Distance_To_Roadways']
features = ['Elevation', 'Horizontal_Distance_To_Roadways', 'Horizontal_Distance_To_Hydrology',
'Horizontal_Distance_To_Fire_Points', 'Aspect','Wilderness_Area1', 'Wilderness_Area4', 'Soil_Type3',
'Soil_Type4','Soil_Type10', 'Soil_Type29',
'Soil_Type38']
x_train, x_test, y_train, y_test = train_test_split(dtrain[features], label, test_size =.3 )
import numpy as np
import pandas as pd
import seaborn as sns
import lightgbm as lgb
import time
from sklearn.model_selection import KFold,StratifiedKFold<load_from_csv>
clf = DecisionTreeClassifier(criterion='gini', max_depth=8, min_samples_split=2, class_weight= None, max_features=None,
max_leaf_nodes=None,
min_impurity_decrease=0.0,
min_impurity_split=None,
min_samples_leaf=1,
min_weight_fraction_leaf=0.0,
presort='deprecated',
random_state=None, )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 print('Rows: ',train_df.shape[0],'Columns: ',train_df.shape[1])
 train_df.info()<count_values>
grid_params = {'criterion' : ["gini", "entropy"]}
grid = GridSearchCV(estimator=clf, param_grid=grid_params, n_jobs=-1, cv = 5)
 grid.fit(x_train, y_train)
 grid.score(x_test, y_test )
train_df['target'].value_counts()<prepare_x_and_y>
grid.best_estimator_
y_pred = grid.predict(x_test )
X_test = test_df.drop('ID_code',axis=1)
 X = train_df.drop(['ID_code','target'],axis=1)
 y = train_df['target']<choose_model_class>
clf.fit(x_train, y_train)
 print(f'No of Leaves : {clf.get_n_leaves() }')
 clf.feature_importances_
n_fold = 5
folds = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=42 )<init_hyperparams>
print(classification_report(y_test, y_pred, labels= list(label_dict.keys()), target_names=list(label_dict.values())) )
params = {'num_leaves': 8,'min_data_in_leaf': 42,'objective': 'binary','max_depth': 16,'learning_rate': 0.0123,'boosting': 'gbdt','bagging_freq': 5,
'bagging_fraction': 0.8,'feature_fraction': 0.8201,'bagging_seed': 11,'reg_alpha': 1.728910519108444,'reg_lambda': 4.9847051755586085,'random_state': 42,
'metric': 'auc','verbosity': -1,'subsample': 0.81,'min_gain_to_split': 0.01077313523861969,'min_child_weight': 19.428902804238373,'num_threads': 4}<train_model>
rnd_clf = RandomForestClassifier()
grid_params_1 = {'max_depth' : [18], 'n_estimators' : [127], 'criterion':['entropy']}
grid = GridSearchCV(estimator=rnd_clf, param_grid=grid_params_1, n_jobs=-1, cv = 5)
 grid.fit(x_train, y_train )
prediction = np.zeros(len(X_test))
for fold_n,(train_index, valid_index)in enumerate(folds.split(X,y)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 model = lgb.train(params,train_data,num_boost_round=20000,
valid_sets = [train_data, valid_data],verbose_eval=300,early_stopping_rounds = 200)
 prediction += model.predict(X_test, num_iteration=model.best_iteration)/5<train_model>
print(grid.best_score_)
 print(grid.score(x_test, y_test))
train_pool = Pool(X,y)
 m = CatBoostClassifier(iterations=300,eval_metric="AUC", boosting_type = 'Ordered')
 m.fit(X,y,silent=True)
 y_pred1 = m.predict(X_test)
 m.best_score_<save_to_csv>
final_clf = RandomForestClassifier(max_depth=18, n_estimators=127, criterion='entropy')
 final_clf.fit(x_train, y_train)
 print(final_clf.score(x_train, y_train))
print(final_clf.score(x_test, y_test))
y_hat = final_clf.predict(x_test )
sub = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub["target"] = prediction
sub.to_csv("submission.csv", index=False )<save_to_csv>
print(classification_report(y_test, y_hat, target_names=label_dict.values()))
sub1 = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub1["target"] = y_pred1
sub1.to_csv("submission1.csv", index=False )<set_options>
imp_feat = pd.DataFrame(index= features , data= final_clf.feature_importances_)
 imp_feat.rename(columns={0 : 'Importance'}, inplace=True)
 imp_feat.sort_values(by='Importance', axis =0, ascending=False )
sns.set(font_scale=1)
 warnings.simplefilter(action='ignore', category=FutureWarning)
 warnings.filterwarnings('ignore' )<load_from_csv>
xgb_clf = XGBClassifier(n_estimators=100, max_depth = 12 )
random_state = 42
np.random.seed(random_state)
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<normalization>
xgb_clf.fit(x_train, y_train)
 xgb_clf.score(x_test, y_test )
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y<init_hyperparams>
y_pred = xgb_clf.predict(x_test )
lgb_params = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : -1,
"num_leaves" : 13,
"learning_rate" : 0.01,
"bagging_freq": 5,
"bagging_fraction" : 0.4,
"feature_fraction" : 0.05,
"min_data_in_leaf": 80,
"min_sum_heassian_in_leaf": 10,
"tree_learner": "serial",
"boost_from_average": "false",
"bagging_seed" : random_state,
"verbosity" : 1,
"seed": random_state}<create_dataframe>
print(classification_report(y_test, y_pred, target_names=label_dict.values()))
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=random_state)
 oof = train[['ID_code', 'target']]
oof['predict'] = 0
predictions = test[['ID_code']]
val_aucs = []
feature_importance = pd.DataFrame()<prepare_x_and_y>
xgb_clf.fit(dtrain[features], label )
features = [col for col in train.columns if col not in ['target', 'ID_code']]
X_test = test[features].values<split>
y_test_hat = xgb_clf.predict(dtest[features] )
<save_to_csv><EOS> 
df_submit = pd.read_csv(submit_path, index_col=0)
 df_submit['Cover_Type'] =y_test_hat
df_submit.to_csv('submit_kaggle.csv' )