Kaggle competition: Forest Cover Type Prediction
def create_model() :
model = Sequential()
model.add(Dense(2**12, activation='relu', input_dim=training_features.shape[1]))
model.add(Dropout(rate=0.5))
model.add(Dense(2**8, activation='relu'))
model.add(Dropout(rate=0.4))
model.add(Dense(2**6, activation='linear'))
model.add(Dropout(rate=0.1))
model.add(Dense(2**4, activation='linear'))
model.add(Dense(1, activation='sigmoid'))
my_opt = optimizers.Adam(lr=0.001, beta_1=0.995, beta_2=0.99999, epsilon=0.01, decay=1e-6, amsgrad=True)
 model.compile(optimizer=my_opt,
loss='binary_crossentropy',
metrics=['binary_accuracy'])
 return model<split>
train_data=pd.read_csv("/kaggle/input/forest-cover-type-prediction/train.csv" )
model_ann = create_model()
result_ann=np.zeros(testing_features.shape[0])
 skf = StratifiedKFold(n_splits=2, random_state=None)
 for counter,(train_index, valid_index)in enumerate(skf.split(training_features_scaled, training_targets),1):
print(counter)
 X_train, y_train = training_features_scaled.iloc[train_index], training_targets.iloc[train_index]
X_valid, y_valid = training_features_scaled.iloc[valid_index], training_targets.iloc[valid_index]
if simple_aug:
X_train, y_train = augment_fast(X_train.values, y_train.values)
 X_train = pd.DataFrame(X_train)
 print("Augmenation done!")
 es = EarlyStopping(monitor='val_binary_accuracy', mode='auto', verbose=0, patience=50)
 cp = ModelCheckpoint(filepath='/best_weights_'+str(counter)+'_.hdf5', verbose=1, save_best_only=True)
 model_ann.fit(X_train, y_train, batch_size=2**8, epochs=1000, validation_data=(X_valid, y_valid), callbacks=[es, cp], verbose=1)
 model_best = create_model()
model_best.load_weights('/best_weights_'+str(counter)+'_.hdf5')
 result_ann += model_best.predict(testing_features_scaled)[:,0]<save_to_csv>
test_data=pd.read_csv("/kaggle/input/forest-cover-type-prediction/test.csv" )
submission_ann = sample_submission.copy()
submission_ann['target'] = result/counter
filename="{:%Y-%m-%d_%H_%M}_ann.csv".format(datetime.now())
 submission_ann.to_csv(filename, index=False )<split>
train_data['Cover_Type'].value_counts()
%%time
result_gnb=np.zeros(testing_features.shape[0])
 skf = StratifiedKFold(n_splits=5, random_state=None)
 for counter,(train_index, valid_index)in enumerate(skf.split(training_features, training_targets),1):
print(counter)
 X_train, y_train = training_features.iloc[train_index], training_targets.iloc[train_index]
X_valid, y_valid = training_features.iloc[valid_index], training_targets.iloc[valid_index]
if simple_aug:
X_train, y_train = augment_fast(X_train.values, y_train.values)
 X_train = pd.DataFrame(X_train)
 print("Augmenation done!")
 skf_inner = StratifiedKFold(n_splits=4, random_state=None)
 def ml_fun(parameters):
parameters = parameters[0]
score = -cross_val_score(GaussianNB(priors=None, var_smoothing = parameters[0]),
X_train, y_train, cv=skf_inner.split(X_train, y_train), scoring='roc_auc', n_jobs=-1 ).mean()
score = np.array(score)
 return score
bounds = [
{'name': 'var_smoothing', 'type': 'continuous', 'domain':(1e-11, 1e-7)}
]
optimizer = GPyOpt.methods.BayesianOptimization(f=ml_fun, domain=bounds,
acquisition_type ='MPI',
exact_eval=True)
 optimizer.run_optimization(1e2, 1e3, verbosity=True)
 gnb = GaussianNB(priors=None, var_smoothing = optimizer.x_opt[0] ).fit(X_train, y_train)
 GaussNB_val_proba = gnb.predict_proba(X_valid)
 fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_valid, GaussNB_val_proba[:, 1])
 print("GaussianNB validation {d}: {:.5f}".format(counter, auc(fpr_keras, tpr_keras)))
 result_gnb += gnb.predict_proba(testing_features )<save_to_csv>
from sklearn.model_selection import train_test_split
submission_gnb = sample_submission.copy()
submission_gnb['target'] = result_gnb/counter
filename="{:%Y-%m-%d_%H_%M}_gbn.csv".format(datetime.now())
 submission_gnb.to_csv(filename, index=False )<split>
from sklearn.model_selection import train_test_split
%%time
result_LogR=np.zeros(testing_features.shape[0])
 skf = StratifiedKFold(n_splits=5, random_state=None)
 for counter,(train_index, valid_index)in enumerate(skf.split(training_features, training_targets),1):
print(counter)
 X_train, y_train = training_features.iloc[train_index], training_targets.iloc[train_index]
X_valid, y_valid = training_features.iloc[valid_index], training_targets.iloc[valid_index]
if simple_aug:
X_train, y_train = augment_fast(X_train.values, y_train.values)
 X_train = pd.DataFrame(X_train)
 print("Augmenation done!")
 skf_inner = StratifiedKFold(n_splits=4, random_state=None)
 def ml_fun(parameters):
parameters = parameters[0]
score = -cross_val_score(LogisticRegression(penalty='l1', dual=False, solver='saga', multi_class='auto',
tol = parameters[0],
C = parameters[1],
max_iter = parameters[2],
n_jobs=-1),
X_train, y_train, cv=skf_inner.split(X_train, y_train), scoring='roc_auc', n_jobs=-1 ).mean()
score = np.array(score)
 return score
bounds = [
{'name': 'var_smoothing', 'type': 'continuous', 'domain':(1e-11, 1e-7)}
]
optimizer = GPyOpt.methods.BayesianOptimization(f=ml_fun, domain=bounds,
acquisition_type ='MPI',
exact_eval=True)
 optimizer.run_optimization(1e2, 1e3, verbosity=True)
 LogR = LogisticRegression(LogisticRegression(penalty='l1', dual=False, solver='saga', multi_class='auto',
tol = optimizer.x_opt[0],
C = optimizer.x_opt[1],
max_iter = optimizer.x_opt[2],
n_jobs=-1 ).fit(X_train, y_train)
 LogR_val_proba = LogR.predict_proba(X_valid)
 fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_valid, LogR_val_proba[:, 1])
 print("GaussianNB validation {d}: {:.5f}".format(counter, auc(fpr_keras, tpr_keras)))
 result_LogR += LogR.predict_proba(testing_features )<save_to_csv>
X=train_data.drop(labels=['Id','Cover_Type'],axis=1 )
submission_LogR = sample_submission.copy()
submission_LogR['target'] = result_LogR/counter
filename="{:%Y-%m-%d_%H_%M}_gbn.csv".format(datetime.now())
 submission_LogR.to_csv(filename, index=False )<import_modules>
y=train_data['Cover_Type']
import numpy as np
import pandas as pd
import gc
import lightgbm as lgb
from sklearn.model_selection import StratifiedKFold
from scipy.stats import rankdata<load_from_csv>
Xtrain,Xval,ytrain,yval=train_test_split(X,y,random_state=40 )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 features = [x for x in train_df.columns if x.startswith("var")] 
<feature_engineering>
rfc=RandomForestClassifier(n_estimators=100 )
for var in features:
if np.corrcoef(train_df['target'], train_df[var])[1][0] < 0:
train_df[var] = train_df[var] * -1
test_df[var] = test_df[var] * -1<count_values>
rfc.fit(Xtrain,ytrain )
var_stats = {}
for var in features:
var_stats[var] = train_df[var].append(test_df[ind][var] ).value_counts()<feature_engineering>
rfc.score(Xval,yval )
def logit(p):
return np.log(p)- np.log(1 - p)
 def var_to_feat(vr, var_stats, feat_id):
new_df = pd.DataFrame()
new_df["var"] = vr.values
new_df["hist"] = pd.Series(vr ).map(var_stats)
 new_df["feature_id"] = feat_id
new_df["var_rank"] = new_df["var"].rank() /200000.
 return new_df.values<feature_engineering>
predict_data=rfc.predict(test_data.drop(labels=['Id'],axis=1))
TARGET = np.array(list(train_df['target'].values)* 200)
 TRAIN = []
var_mean = {}
var_var = {}
for var in features:
tmp = var_to_feat(train_df[var], var_stats[var], int(var[4:]))
var_mean[var] = np.mean(tmp[:,0])
 var_var[var] = np.var(tmp[:,0])
 tmp[:,0] =(tmp[:,0]-var_mean[var])/var_var[var]
TRAIN.append(tmp)
 TRAIN = np.vstack(TRAIN)
 del train_df
_=gc.collect()
print(TRAIN.shape, len(TARGET))<choose_model_class>
sub=pd.DataFrame(data=predict_data,columns=['Cover_type'] )
model = lgb.LGBMClassifier(**{
'learning_rate': 0.04,
'num_leaves': 31,
'max_bin': 1023,
'min_child_samples': 1000,
'reg_alpha': 0.1,
'reg_lambda': 0.2,
'feature_fraction': 1.0,
'bagging_freq': 1,
'bagging_fraction': 0.85,
'objective': 'binary',
'n_jobs': -1,
'n_estimators':200,})
 MODELS = []
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=11111)
 for fold_,(train_indexes, valid_indexes)in enumerate(skf.split(TRAIN, TARGET)) :
print('Fold:', fold_)
 model = model.fit(TRAIN[train_indexes], TARGET[train_indexes],
eval_set =(TRAIN[valid_indexes], TARGET[valid_indexes]),
verbose = 10,
eval_metric='auc',
early_stopping_rounds=25,
categorical_feature = [2])
 MODELS.append(model)
 del TRAIN, TARGET
_=gc.collect()<save_to_csv>
sub['Id']=test_data['Id']
ypred = np.zeros(( 200000,200))
for feat,var in enumerate(features):
tmp = var_to_feat(test_df[var], var_stats[var], int(var[4:]))
tmp[:,0] =(tmp[:,0]-var_mean[var])/var_var[var]
for model_id in range(10):
model = MODELS[model_id]
ypred[:,feat] += model.predict_proba(tmp)[:,1] / 10.
 ypred = np.mean(logit(ypred), axis=1)
 sub = test_df[['ID_code']]
sub['target'] = ypred
sub['target'] = sub['target'].rank() / 200000.
 sub.to_csv('golden_sub.csv', index=False)
 print(sub.head(10)) 
<import_modules>
sub.set_index('Id',inplace=True )
import lightgbm as lgb
import pandas as pd
import numpy as np
from datetime import datetime
from sklearn.preprocessing import MinMaxScaler
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt<prepare_x_and_y>
sub.to_csv('Sub.csv' )