Kaggle competition: Forest Cover Type Prediction
factors_train = pd.DataFrame(factors_train)
 factors_test = pd.DataFrame(factors_test )<merge>
import pandas as pd
import numpy as np
from sklearn.metrics import confusion_matrix
from sklearn.metrics import f1_score
from sklearn.metrics import accuracy_score
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import r2_score
from sklearn.preprocessing import PolynomialFeatures
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestClassifier
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from lightgbm import LGBMClassifier
from sklearn.linear_model import Ridge
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
train_combined = train.merge(factors_train, left_index = True, right_index = True)
 test_combined = X_test.merge(factors_test, left_index = True, right_index = True )<concatenate>
filepath= '/kaggle/input/forest-cover-type-prediction/train.csv'
filepath1= '/kaggle/input/forest-cover-type-prediction/test.csv'
testdata= pd.read_csv(filepath1)
 testdata2=testdata
traindata= pd.read_csv(filepath)
 traindata.head()
def augment(train, num_n = 1, num_p = 2):
newtrain=[train]
n = train[train.target == 0]
for i in range(num_n):
newtrain.append(n.apply(lambda x:x.values.take(np.random.permutation(len(n)))))
 p = train[train.target == 1]
for i in range(num_p):
newtrain.append(p.apply(lambda x:x.values.take(np.random.permutation(len(p)))))
 return pd.concat(newtrain )<init_hyperparams>
traindata=traindata.drop('Id',axis=1)
 testdata=testdata.drop('Id',axis=1 )
param = {
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.041,
'learning_rate': 0.009,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 70,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 15,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': -1
}<train_model>
y=traindata['Cover_Type']
x=traindata.drop('Cover_Type',axis=1)
 x_train, x_test, y_train, y_test = train_test_split(x, y, train_size=0.70,test_size=0.30, random_state=0 )
train_augmented = augment(train_combined)
 train_data = lgb.Dataset(train_augmented.drop('target', axis = 1), train_augmented['target'])
 model = lgb.train(param, train_data, 19000 )<predict_on_test>
linearRegressor = LinearRegression()
linearRegressor.fit(x_train, y_train)
 y_predicted = linearRegressor.predict(x_test)
 mse = mean_squared_error(y_test, y_predicted)
 r = r2_score(y_test, y_predicted)
 mae = mean_absolute_error(y_test,y_predicted)
 print("Mean Squared Error:",mse)
 print("R score:",r)
 print("Mean Absolute Error:",mae )
prediction = model.predict(test_combined )<save_to_csv>
rf = RandomForestClassifier()
rf.fit(x_train,y_train);
y_predicted_r = rf.predict(x_test)
 mse = mean_squared_error(y_test, y_predicted_r)
 r = r2_score(y_test, y_predicted_r)
 mae = mean_absolute_error(y_test,y_predicted_r)
 print("Mean Squared Error:",mse)
 print("R score:",r)
 print("Mean Absolute Error:",mae )
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = prediction
filename = "sub_{:%Y-%m-%d_%H_%M}.csv".format(datetime.now())
 submission.to_csv(filename, index=False )<import_modules>
regressor = DecisionTreeRegressor(random_state = 0)
 regressor.fit(x_train, y_train)
 y_predicted_d = regressor.predict(x_test)
 mse = mean_squared_error(y_test, y_predicted_d)
 r = r2_score(y_test, y_predicted_d)
 mae = mean_absolute_error(y_test,y_predicted_d)
 print("Mean Squared Error:",mse)
 print("R score:",r)
 print("Mean Absolute Error:",mae )
import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
import lightgbm as lgb
from keras.models import Sequential
from keras.layers import Dense, Dropout, BatchNormalization, Activation
from keras import backend as K
from keras import optimizers
from keras.callbacks import EarlyStopping
from sklearn.preprocessing import StandardScaler
from plotly.offline import init_notebook_mode, iplot
import plotly.graph_objs as go
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold,StratifiedKFold
from sklearn.metrics import roc_auc_score
from gplearn.genetic import SymbolicTransformer<load_from_csv>
polynomial_features= PolynomialFeatures(degree=2)
 x_poly = polynomial_features.fit_transform(x_train)
 x_poly_test = polynomial_features.fit_transform(x_test)
 model = LinearRegression()
model.fit(x_poly, y_train)
 y_predicted_p = model.predict(x_poly_test)
 mse = mean_squared_error(y_test, y_predicted_p)
 r = r2_score(y_test, y_predicted_p)
 mae = mean_absolute_error(y_test,y_predicted_p)
 print("Mean Squared Error:",mse)
 print("R score:",r)
 print("Mean Absolute Error:",mae )
data_train = pd.read_csv('.. /input/train.csv')
 data_test = pd.read_csv('.. /input/test.csv')
 print('Training data shape:',data_train.shape)
 print('Testing data shape:',data_test.shape)
 data_train.head(5 )<feature_engineering>
ridgereg = Ridge(normalize=True)
 ridgereg.fit(x_train, y_train)
 y_pred = ridgereg.predict(x_test)
 mse = mean_squared_error(y_test, y_pred)
 r = r2_score(y_test, y_pred)
 mae = mean_absolute_error(y_test,y_pred)
 print("Mean Squared Error:",mse)
 print("R score:",r)
 print("Mean Absolute Error:",mae )
fe_train = data_train.copy()
fe_test = data_test.copy()
sd = True
st = True
sp = False
gp = True
if st:
idx = features = fe_train.columns.values[2:202]
for df in [fe_train, fe_test]:
df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1)
 if sd:
scaler = StandardScaler()
fe_train.iloc[:,2:] = scaler.fit_transform(fe_train.iloc[:,2:])
 fe_test.iloc[:,1:] = scaler.fit_transform(fe_test.iloc[:,1:])
 if gp:
function_set = ['add', 'sub', 'mul', 'div', 'log', 'sqrt', 'abs', 'neg', 'max', 'min']
generations = 80
population_size = 2000
gp = SymbolicTransformer(generations=generations, population_size=population_size,
hall_of_fame=100, n_components=10,
function_set=function_set,
parsimony_coefficient=0.0005,
max_samples=0.9, verbose=1,
random_state=0, n_jobs=3)
 gp.fit(fe_train.iloc[:,2:],fe_train['target'])
 gp_feature_name = ['NV'+str(i)for i in range(1, 11)]
train_gp_features = pd.DataFrame(gp.transform(fe_train.iloc[:,2:]), columns=gp_feature_name)
 test_gp_features = pd.DataFrame(gp.transform(fe_test.iloc[:,1:]), columns=gp_feature_name)
 if sd:
train_gp_features = pd.DataFrame(scaler.fit_transform(train_gp_features))
test_gp_features = pd.DataFrame(scaler.fit_transform(test_gp_features))
fe_train = pd.concat([fe_train, train_gp_features], axis=1)
 fe_test = pd.concat([fe_test, test_gp_features], axis=1)
 if sp:
fe_zeros = fe_train[fe_train['target'] == 0].sample(frac=0.25)
 fe_ones = fe_train[fe_train['target'] > 0]
fe_train = pd.concat([fe_ones, fe_zeros] ).sample(frac=1)
 X = fe_train.iloc[:,2:]
Y = fe_train['target']
X_target = fe_test.iloc[:,1:]
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2, random_state=113)
<init_hyperparams>
lgb_clf = LGBMClassifier(random_state=17)
 lgb_clf.fit(x_train, y_train)
 y_pred = lgb_clf.predict(x_test)
 mse = mean_squared_error(y_test, y_pred)
 r = r2_score(y_test, y_pred)
 mae = mean_absolute_error(y_test,y_pred)
 print("Mean Squared Error:",mse)
 print("R score:",r)
 print("Mean Absolute Error:",mae )
cv = True
param = {
'bagging_freq': 8,
'bagging_fraction': 0.3,
'boost_from_average':False,
'boost': 'gbdt',
'feature_fraction': 0.2,
'learning_rate': 0.008,
'max_depth':2,
'metric':'auc',
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'xentropy',
'verbosity':1,
"bagging_seed" : 122,
"seed": 20,
}
num_boost_round = 200000
verbose_eval=5000
if not cv:
lgbm_train = lgb.Dataset(X_train,label=y_train)
 lgbm_test = lgb.Dataset(X_test,label=y_test)
 lgbm_valid =(lgbm_test,lgbm_train)
 valid_names = ['valid','train']
lgbm_model = lgb.train(param,lgbm_train,num_boost_round=num_boost_round,valid_sets=lgbm_valid,valid_names=valid_names,
verbose_eval=verbose_eval,
early_stopping_rounds=5000)
 lgbm_predict = lgbm_model.predict(X_target)
 else:
lgbm_cv_valid_score = []
lgbm_cv_predict = []
kf = StratifiedKFold(n_splits=5,shuffle = False, random_state=311)
 for _fold,(trn_idx, val_idx)in enumerate(kf.split(X.values, Y.values)) :
print('Fold{}:'.format(_fold+1))
X_cv_train, y_cv_train = X.iloc[trn_idx], Y.iloc[trn_idx]
X_cv_test, y_cv_test = X.iloc[val_idx], Y.iloc[val_idx]
lgbm_cv_train = lgb.Dataset(X_cv_train,label=y_cv_train)
 lgbm_cv_test = lgb.Dataset(X_cv_test,label=y_cv_test)
 lgbm_cv_valid =(lgbm_cv_test,lgbm_cv_train)
 valid_names = ['valid','train']
lgbm_model = lgb.train(param,lgbm_cv_train,num_boost_round=num_boost_round,valid_sets=lgbm_cv_valid,valid_names=valid_names,
verbose_eval=verbose_eval,
early_stopping_rounds=5000)
 lgbm_cv_valid_score.append(roc_auc_score(y_cv_test,lgbm_model.predict(X_cv_test)))
 print('\tCVscore:{}'.format(lgbm_cv_valid_score[_fold]))
print('-'*50)
 lgbm_cv_predict.append(lgbm_model.predict(X_target))
print('---Mean CV score:{}---'.format(np.mean(lgbm_cv_valid_score)))
 lgbm_predict = np.mean(lgbm_cv_predict,axis = 0 )<train_on_grid>
param_grid = {'num_leaves': [7, 15, 31, 63],
'max_depth': [3, 4, 5, 6, -1]}
grid_searcher = GridSearchCV(estimator=lgb_clf, param_grid=param_grid,
cv=5, verbose=1, n_jobs=4)
 grid_searcher.fit(x_train, y_train)
 mse = mean_squared_error(y_test, y_pred)
 r = r2_score(y_test, y_pred)
 mae = mean_absolute_error(y_test,y_pred)
 print("Mean Squared Error:",mse)
 print("R score:",r)
 print("Mean Absolute Error:",mae )
def auc(y_true, y_pred):
auc = tf.metrics.auc(y_true, y_pred)[1]
K.get_session().run(tf.local_variables_initializer())
 return auc
input_dim = X_train.shape[1]
epochs = 100
model = Sequential()
model.add(Dense(128,input_dim=input_dim, kernel_initializer='normal'))
model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Dense(1, activation='sigmoid'))
adam = optimizers.Adam(lr=0.001)
 model.compile(optimizer=adam,loss='binary_crossentropy',metrics = [auc])
 nn_model = model
nn_history = nn_model.fit(X_train, y_train, validation_data =(X_test ,y_test),
epochs=epochs,batch_size=64,verbose=1)
 nn_predict = nn_model.predict(X_target)[:,0]<save_to_csv>
rf = RandomForestClassifier()
rf.fit(x,y);
Prediction = rf.predict(testdata )
<load_from_csv><EOS> 
predictionlist=Prediction.tolist()
Passengerid=testdata2['Id'].tolist()
output=pd.DataFrame(list(zip(Passengerid, predictionlist)) ,
columns=['Id','Cover_type'])
 output.head()
output.to_csv('my_submission(ForestCoverTypePrediction ).csv', index=False )