Kaggle competition: Forest Cover Type Prediction
<SOS> metric: CategorizationAccuracy Kaggle data source: forest-cover-type-prediction<categorify>
warnings.filterwarnings('ignore')
 dataset = pandas.read_csv(".. /input/train.csv")
 dataset = dataset.iloc[:,1:]
def augment(train, target, num_n=1, num_p=2):
newtrain=[train]
newtarget=[target]
n=train[target==0]
z=target[target==0]
for i in range(num_n):
random_state = np.random.permutation(len(n))
newtrain.append(n.apply(lambda x:x.values.take(random_state)))
 newtarget.append(z.apply(lambda x:x.values.take(random_state)))
 for i in range(num_p):
p=train[target>0]
o=target[target>0]
random_state = np.random.permutation(len(p))
newtrain.append(p.apply(lambda x:x.values.take(random_state)))
 newtarget.append(o.apply(lambda x:x.values.take(random_state)))
 return pd.concat(newtrain), pd.concat(newtarget )<categorify>
dataset.groupby('Cover_Type' ).size()
def augment_fast(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
mask = mask[:,0]
x1 = x[mask,:].copy()
ids = np.arange(x1.shape[0])
 for p in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,p] = x1[ids][:,p]
xs.append(x1)
 for i in range(t//2):
mask = y==0
mask = mask[:,0]
x1 = x[mask, :].copy()
ids = np.arange(x1.shape[0])
 for n in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,n] = x1[ids][:,n]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y[:,0],ys,yn])
 return x,y<define_variables>
cols = dataset.columns
r,c = dataset.shape
data = pandas.DataFrame(index=numpy.arange(0, r),columns=['Wilderness_Area','Soil_Type','Cover_Type'])
 for i in range(0,r):
w=0;
s=0;
for j in range(10,14):
if(dataset.iloc[i,j] == 1):
w=j-9
break
for k in range(14,54):
if(dataset.iloc[i,k] == 1):
s=k-13
break
data.iloc[i]=[w,s,dataset.iloc[i,c-1]]
sns.countplot(x="Wilderness_Area", hue="Cover_Type", data=data)
 plt.show()
plt.rc("figure", figsize=(25, 10))
sns.countplot(x="Soil_Type", hue="Cover_Type", data=data)
 plt.show()
training_features = original_features
training_targets = original_targets<define_variables>
rem = []
for c in dataset.columns:
if dataset[c].std() == 0:
rem.append(c)
 dataset.drop(rem,axis=1,inplace=True)
 print(rem)
simple_aug = False<train_model>
r, c = dataset.shape
cols = dataset.columns
i_cols = []
for i in range(0,c-1):
i_cols.append(i)
 ranks = []
array = dataset.values
X = array[:,0:(c-1)]
Y = array[:,(c-1)]
val_size = 0.1
seed = 0
X_train, X_val, Y_train, Y_val = cross_validation.train_test_split(X, Y, test_size=val_size, random_state=seed)
 X_all = []
X_all_add =[]
rem = []
i_rem = []
comb = []
comb.append("All+1.0")
 X_all.append(['Orig','All', X_train,X_val,1.0,cols[:c-1],rem,ranks,i_cols,i_rem])
 size=10
X_temp = StandardScaler().fit_transform(X_train[:,0:size])
 X_val_temp = StandardScaler().fit_transform(X_val[:,0:size])
 X_con = numpy.concatenate(( X_temp,X_train[:,size:]),axis=1)
 X_val_con = numpy.concatenate(( X_val_temp,X_val[:,size:]),axis=1)
 X_all.append(['StdSca','All', X_con,X_val_con,1.0,cols,rem,ranks,i_cols,i_rem])
 X_temp = MinMaxScaler().fit_transform(X_train[:,0:size])
 X_val_temp = MinMaxScaler().fit_transform(X_val[:,0:size])
 X_con = numpy.concatenate(( X_temp,X_train[:,size:]),axis=1)
 X_val_con = numpy.concatenate(( X_val_temp,X_val[:,size:]),axis=1)
 X_all.append(['MinMax', 'All', X_con,X_val_con,1.0,cols,rem,ranks,i_cols,i_rem])
 X_temp = Normalizer().fit_transform(X_train[:,0:size])
 X_val_temp = Normalizer().fit_transform(X_val[:,0:size])
 X_con = numpy.concatenate(( X_temp,X_train[:,size:]),axis=1)
 X_val_con = numpy.concatenate(( X_val_temp,X_val[:,size:]),axis=1)
 X_all.append(['Norm', 'All', X_con,X_val_con,1.0,cols,rem,ranks,i_cols,i_rem])
 trans_list = []
for trans,name,X,X_val,v,cols_list,rem_list,rank_list,i_cols_list,i_rem_list in X_all:
trans_list.append(trans )
print('Training feature shape: ', training_features.shape)
 print('Training targe shape: ', training_targets.shape)
 print('Testing targe shape: ', testing_features.shape )<init_hyperparams>
ratio_list = [0.75,0.50,0.25]
param = {
'bagging_freq': 5,
'bagging_fraction': 0.335,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.041,
'learning_rate': 0.1,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': -1,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': -1
}<split>
feat = []
feat_list =[]
model = LogisticRegression(random_state=seed,n_jobs=-1)
 n = 'RFE'
feat_list.append(n)
 for val in ratio_list:
comb.append("%s+%s" %(n,val))
feat.append([n,val,RFE(model,val*(c-1)) ])
 for trans,s, X, X_val, d, cols, rem, ra, i_cols, i_rem in X_all:
for name,v, model in feat:
model.fit(X,Y_train)
 joined = []
for i, pred in enumerate(list(model.ranking_)) :
joined.append([i,cols[i],pred])
 joined_sorted = sorted(joined, key=lambda x: x[2])
 rem_start = int(( v*(c-1)))
 cols_list = []
i_cols_list = []
rank_list =[]
rem_list = []
i_rem_list = []
for i, col, j in joined_sorted:
rank_list.append([i,j-1])
 if(( j-1)< rem_start):
cols_list.append(col)
 i_cols_list.append(i)
 else:
rem_list.append(col)
 i_rem_list.append(i)
 X_all_add.append([trans,name,X,X_val,v,cols_list,rem_list,[x[1] for x in sorted(rank_list,key=lambda x:x[0])],i_cols_list,i_rem_list])
 plt.rc("figure", figsize=(25, 10))
for f_name in feat_list:
leg=[]
fig, ax = plt.subplots()
for trans,name,X,X_val,v,cols_list,rem_list,rank_list,i_cols_list,i_rem_list in X_all_add:
if(name==f_name):
plt.plot(rank_list)
 leg.append(trans+"+"+name+"+%s"% v)
 ax.set_xticks(range(c-1))
ax.set_xticklabels(cols[:c-1],rotation='vertical')
 plt.legend(leg,loc='best')
 plt.show()
%%time
result=np.zeros(testing_features.shape[0])
 skf = StratifiedKFold(n_splits=5, random_state=None)
 for counter,(train_index, valid_index)in enumerate(skf.split(training_features, training_targets),1):
print(counter)
 X_train, y_train = training_features.iloc[train_index], training_targets.iloc[train_index]
X_valid, y_valid = training_features.iloc[valid_index], training_targets.iloc[valid_index]
if simple_aug:
X_train, y_train = augment_fast(X_train.values, y_train.values)
 X_train = pd.DataFrame(X_train)
 print("Augmenation done!")
 lgb_train = lgb.Dataset(X_train, label=y_train)
 lgb_val = lgb.Dataset(X_valid, label=y_valid)
 model_lgb = lgb.train(param, lgb_train, 1000, valid_sets = [lgb_train, lgb_val], verbose_eval=20, early_stopping_rounds = 50)
 result += model_lgb.predict(testing_features, num_iteration=model_lgb.best_iteration )<save_to_csv>
feat = []
feat_list =[]
n = 'SelK'
feat_list.append(n)
 for val in ratio_list:
comb.append("%s+%s" %(n,val))
feat.append([n,val,SelectPercentile(score_func=f_classif,percentile=val*100)])
 for trans,s, X, X_val, d, cols, rem, ra, i_cols, i_rem in X_all:
for name,v, model in feat:
model.fit(X,Y_train)
 joined = []
for i, pred in enumerate(list(model.scores_)) :
joined.append([i,cols[i],pred])
 joined_sorted = sorted(joined, key=lambda x: -x[2])
 rem_start = int(( v*(c-1)))
 cols_list = []
i_cols_list = []
rank_list =[]
rem_list = []
i_rem_list = []
for j,(i, col, x)in enumerate(list(joined_sorted)) :
rank_list.append([i,j])
 if(j < rem_start):
cols_list.append(col)
 i_cols_list.append(i)
 else:
rem_list.append(col)
 i_rem_list.append(i)
 X_all_add.append([trans,name,X,X_val,v,cols_list,rem_list,[x[1] for x in sorted(rank_list,key=lambda x:x[0])],i_cols_list,i_rem_list])
 plt.rc("figure", figsize=(25, 10))
for f_name in feat_list:
leg=[]
fig, ax = plt.subplots()
for trans,name,X,X_val,v,cols_list,rem_list,rank_list,i_cols_list,i_rem_list in X_all_add:
if(name==f_name):
plt.plot(rank_list)
 leg.append(trans+"+"+name+"+%s"% v)
 ax.set_xticks(range(c-1))
ax.set_xticklabels(cols[:c-1],rotation='vertical')
 plt.legend(leg,loc='best')
 plt.show()
submission_lgb = sample_submission.copy()
submission_lgb['target'] = result/counter
filename="{:%Y-%m-%d_%H_%M}_lgb.csv".format(datetime.now())
 submission_lgb.to_csv(filename, index=False )<normalization>
rank_df = pandas.DataFrame(data=[x[7] for x in X_all_add],columns=cols[:c-1])
 _ = rank_df.boxplot(rot=90)
scaler = MinMaxScaler()
scaler.fit(training_features)
 training_features_scaled = pd.DataFrame(scaler.transform(training_features))
testing_features_scaled = pd.DataFrame(scaler.transform(testing_features))<choose_model_class>
rank_df = pandas.DataFrame(data=[x[7] for x in X_all_add],columns=cols[:c-1])
 med = rank_df.median()
print(med)
 with open("median.csv", "w")as subfile:
subfile.write("Column,Median
")
 subfile.write(med.to_string() )