Kaggle competition: Forest Cover Type Prediction
<SOS> metric: CategorizationAccuracy Kaggle data source: forest-cover-type-prediction<choose_model_class>
!pip install xgboost
learn = cnn_learner(data, models.densenet161, metrics=error_rate, model_dir="/tmp/model/" )<train_model>
!pip install feature-engine
learn.fit_one_cycle(6 )<train_model>
!pip install lazypredict
learn.fit_one_cycle(5, max_lr=slice(3e-5,3e-4))<choose_model_class>
data = pd.read_csv('.. /input/forest-cover-type-prediction/train.csv')
 data.head(5 )
interp = ClassificationInterpretation.from_learner(learn )<predict_on_test>
class DistanceTransformer(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X):
X = X.copy()
X['EuclideanDistanceHidroloy'] = np.around(
 np.sqrt(X['Horizontal_Distance_To_Hydrology'] **2 +
X['Vertical_Distance_To_Hydrology'] **2),
4)
 X['Elevation_Vertical_Hydro_Minus'] = X['Elevation'] - X['Vertical_Distance_To_Hydrology']
X['Elevation_Vertical_Hydro_Plus'] = X['Elevation'] + X['Vertical_Distance_To_Hydrology']
X['Elevation_Vertical_Hydro_Prod'] = X['Elevation'] * X['Vertical_Distance_To_Hydrology']
X['Elevation_Horizontal_Hydro_Minus'] = X['Elevation'] - X['Horizontal_Distance_To_Hydrology']
X['Elevation_Horizontal_Hydro_Plus'] = X['Elevation'] + X['Horizontal_Distance_To_Hydrology']
X['Elevation_Horizontal_Hydro_Prod'] = X['Elevation'] * X['Horizontal_Distance_To_Hydrology']
X['Elevation_Horizontal_Fire_Minus'] = X['Elevation'] - X['Horizontal_Distance_To_Fire_Points']
X['Elevation_Horizontal_Fire_Plus'] = X['Elevation'] + X['Horizontal_Distance_To_Fire_Points']
X['Elevation_Horizontal_Fire_Prod'] = X['Elevation'] * X['Horizontal_Distance_To_Fire_Points']
X['Elevation_Horizontal_Roadways_Minus'] = X['Elevation'] - X['Horizontal_Distance_To_Roadways']
X['Elevation_Horizontal_Roadways_Plus'] = X['Elevation'] + X['Horizontal_Distance_To_Roadways']
X['Elevation_Horizontal_Roadways_Prod'] = X['Elevation'] * X['Horizontal_Distance_To_Roadways']
X['Hidrology_Horizonal_Fire_Minus'] = X['Horizontal_Distance_To_Hydrology'] - X['Horizontal_Distance_To_Fire_Points']
X['Hidrology_Horizonal_Fire_Plus'] = X['Horizontal_Distance_To_Hydrology'] + X['Horizontal_Distance_To_Fire_Points']
X['Hidrology_Horizonal_Fire_Prod'] = X['Horizontal_Distance_To_Hydrology'] * X['Horizontal_Distance_To_Fire_Points']
X['Hidrology_Horizonal_Roadways_Minus'] = X['Horizontal_Distance_To_Hydrology'] - X['Horizontal_Distance_To_Roadways']
X['Hidrology_Horizonal_Roadways_Plus'] = X['Horizontal_Distance_To_Hydrology'] + X['Horizontal_Distance_To_Roadways']
X['Hidrology_Horizonal_Roadways_Prod'] = X['Horizontal_Distance_To_Hydrology'] * X['Horizontal_Distance_To_Roadways']
X['Hidrology_Horizonal_Vertical_Minus'] = X['Horizontal_Distance_To_Hydrology'] - X['Vertical_Distance_To_Hydrology']
X['Hidrology_Horizonal_Vertical_Plus'] = X['Horizontal_Distance_To_Hydrology'] + X['Vertical_Distance_To_Hydrology']
X['Hidrology_Horizonal_Vertical_Prod'] = X['Horizontal_Distance_To_Hydrology'] * X['Vertical_Distance_To_Hydrology']
X['Hidrology_Vertical_Fire_Minus'] = X['Vertical_Distance_To_Hydrology'] - X['Horizontal_Distance_To_Fire_Points']
X['Hidrology_Vertical_Fire_Plus'] = X['Vertical_Distance_To_Hydrology'] + X['Horizontal_Distance_To_Fire_Points']
X['Hidrology_Vertical_Fire_Prod'] = X['Vertical_Distance_To_Hydrology'] * X['Horizontal_Distance_To_Fire_Points']
X['Hidrology_Vertical_Fire_Minus'] = X['Vertical_Distance_To_Hydrology'] - X['Horizontal_Distance_To_Roadways']
X['Hidrology_Vertical_Fire_Plus'] = X['Vertical_Distance_To_Hydrology'] + X['Horizontal_Distance_To_Roadways']
X['Hidrology_Vertical_Fire_Prod'] = X['Vertical_Distance_To_Hydrology'] * X['Horizontal_Distance_To_Roadways']
X.drop(['Horizontal_Distance_To_Hydrology', 'Vertical_Distance_To_Hydrology', 'Horizontal_Distance_To_Roadways', 'Horizontal_Distance_To_Fire_Points'], axis=1, inplace=True)
 return X
a,b,c=learn.predict(open_image(".. /input/test/test/"+"000940378805c44108d287872b2f04ce.jpg"))
print(c)
 print(c[1].numpy() )<predict_on_test>
class DropIdentifierFeatures(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y=None):
return self
def transform(self, X):
X = X.copy()
X.drop('Id', axis=1, inplace=True)
 return X
def pred(name):
a,b,c=learn.predict(open_image(".. /input/test/test/"+name))
return c[1].numpy()<feature_engineering>
class FromDummiesToCategories(BaseEstimator, TransformerMixin):
def __init__(self, cols_to_operate, new_column_name):
self.cols_to_operate = cols_to_operate
self.new_column_name = new_column_name
def fit(self, X, y=None):
return self
def transform(self, X):
X = X.copy()
X1 = pd.DataFrame(X[self.cols_to_operate])
 serie = X1.columns[np.where(X1!=0)[1]]
X[self.new_column_name] = serie
X.drop(self.cols_to_operate, axis=1, inplace=True)
 return X
test_df["has_cactus"]=test_df["id"].apply(lambda x:pred(x))<save_to_csv>
class AspectTransformer(BaseEstimator, TransformerMixin):
def __init__(self, column, new_name):
self.column = column
self.new_name = new_name
def fit(self, X, y=None):
return self
def transform(self, X):
X = X.copy()
X[self.new_name] = X[self.column].copy().apply(self._reclassify_aspect)
 X.drop(self.column, axis=1, inplace=True)
 return X
def _reclassify_aspect(self, x):
if x<0:
return 'Flat'
if 0 >= x < 45:
return 'North'
if 45 >= x < 90:
return 'North_East'
if 90 >= x < 135:
return 'East'
if 135 >= x < 180:
return 'South_East'
if 180 >= x < 225:
return 'South'
if 225 >= x < 270:
return 'South_West'
if 270 >= x < 315:
return 'West'
if 315 >= x <360:
return 'North_West'
if 360 >= x :
return 'North_West'
test_df.to_csv('submission.csv',index=False )<set_options>
sss = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)
 X = data.drop('Cover_Type', axis=1 ).copy()
y = data['Cover_Type'].copy()
for train_index, test_index in sss.split(X, y):
X_train, X_test = X.iloc[train_index], X.iloc[test_index]
y_train, y_test = y.iloc[train_index], y.iloc[test_index]
%matplotlib inline
%reload_ext autoreload
%autoreload 2<set_options>
soil_columns = [x for x in X.columns if x.startswith('Soil_Type')]
wilder_columns = [x for x in X.columns if x.startswith('Wilder')]
np.random.seed(42 )<load_from_csv>
from feature_engine.discretisation import EqualWidthDiscretiser, EqualFrequencyDiscretiser
from feature_engine.selection import DropConstantFeatures, DropDuplicateFeatures, DropCorrelatedFeatures, SelectBySingleFeaturePerformance, RecursiveFeatureElimination, SmartCorrelatedSelection, DropFeatures
from feature_engine.encoding import RareLabelEncoder, OneHotEncoder, OrdinalEncoder
from feature_engine.outliers import Winsorizer
from feature_engine.creation import MathematicalCombination
from sklearn.pipeline import Pipeline
data_folder = Path(".. /input")
 train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/sample_submission.csv" )<choose_model_class>
pipeline_list_1 = [ 
('dropuniquefeatures', DropIdentifierFeatures()), 
('aspect', AspectTransformer(column='Aspect', new_name='Orientation')) , 
('soil_columns_dummies', FromDummiesToCategories(cols_to_operate=soil_columns, new_column_name='Soil_Type')) , 
('wilder_columns_dummies', FromDummiesToCategories(cols_to_operate=wilder_columns, new_column_name='Wilderness')) , 
('soil_rare', RareLabelEncoder(tol=0.05, variables=['Soil_Type'])) , 
('dp', DropConstantFeatures(tol=0.99)) , 
('dd', DropDuplicateFeatures()), 
('dt', DistanceTransformer()), 
('one_hot',OneHotEncoder(variables=['Soil_Type', 'Wilderness', 'Orientation'])) , 
('dteq', EqualFrequencyDiscretiser(q=10, variables=['EuclideanDistanceHidroloy',
'Hillshade_9am', 'Hillshade_Noon', 'Hillshade_3pm'])) , 
('dcf', DropCorrelatedFeatures(variables=None, method='pearson', threshold=0.80)) ,
]
pipeline_1 = Pipeline(pipeline_list_1)
 X_train_pipe_1 = pipeline_1.fit_transform(X_train)
 X_train_pipe_1.head()
learn50 = cnn_learner(train_img, models.densenet161, metrics=[error_rate, accuracy], model_dir="/tmp/model/" )<train_model>
X_test_pipe_1 = pipeline_1.transform(X_test)
 X_test_pipe_1.head()
lr = 3e-02
learn50.fit_one_cycle(5 , slice(lr))<predict_on_test>
pipeline_list_2 = [ 
('dropuniquefeatures', DropIdentifierFeatures()), 
('aspect', AspectTransformer(column='Aspect', new_name='Orientation')) , 
('soil_columns_dummies', FromDummiesToCategories(cols_to_operate=soil_columns, new_column_name='Soil_Type')) , 
('wilder_columns_dummies', FromDummiesToCategories(cols_to_operate=wilder_columns, new_column_name='Wilderness')) , 
('soil_rare', RareLabelEncoder(tol=0.05, variables=['Soil_Type'])) , 
('dp', DropConstantFeatures(tol=0.99)) , 
('dd', DropDuplicateFeatures()), 
('dt', DistanceTransformer()), 
('one_hot',OneHotEncoder(variables=['Soil_Type', 'Wilderness', 'Orientation'])) , 
('winds', Winsorizer(variables=['EuclideanDistanceHidroloy', 'Elevation',
'Hillshade_9am', 'Hillshade_Noon', 'Hillshade_3pm'])) , 
('dcf', DropCorrelatedFeatures(variables=None, method='pearson', threshold=0.80)) ,
]
pipeline_2 = Pipeline(pipeline_list_2)
 X_train_pipe_2 = pipeline_2.fit_transform(X_train)
 X_train_pipe_2.head()
preds,_ = learn50.get_preds(ds_type=DatasetType.Test )<save_to_csv>
pipeline_list_3 = [ 
('dropuniquefeatures', DropIdentifierFeatures()), 
('aspect', AspectTransformer(column='Aspect', new_name='Orientation')) , 
('soil_columns_dummies', FromDummiesToCategories(cols_to_operate=soil_columns, new_column_name='Soil_Type')) , 
('wilder_columns_dummies', FromDummiesToCategories(cols_to_operate=wilder_columns, new_column_name='Wilderness')) , 
('soil_rare', RareLabelEncoder(tol=0.05, variables=['Soil_Type'])) , 
('dp', DropConstantFeatures(tol=0.99)) , 
('dt', DistanceTransformer()), 
('one_hot',OneHotEncoder(variables=['Soil_Type', 'Wilderness', 'Orientation'])) , 
('winds', Winsorizer(variables=['EuclideanDistanceHidroloy', 'Elevation',
'Hillshade_9am', 'Hillshade_Noon', 'Hillshade_3pm'])) , 
('dd', SmartCorrelatedSelection()),
]
pipeline_3 = Pipeline(pipeline_list_3)
 X_train_pipe_3 = pipeline_3.fit_transform(X_train)
 X_train_pipe_3.head()
test_df.to_csv('submission.csv', index=False )<import_modules>
pipeline_list_4 = [ 
('dropuniquefeatures', DropIdentifierFeatures()), 
('dropwild_soil',DropFeatures(soil_columns + wilder_columns)) , 
('aspect', AspectTransformer(column='Aspect', new_name='Orientation')) , 
('dp', DropConstantFeatures(tol=0.99)) , 
('dt', DistanceTransformer()), 
('one_hot',OneHotEncoder(variables=['Orientation'])) , 
('winds', Winsorizer(variables=[
'EuclideanDistanceHidroloy', 'Elevation', 'Hillshade_9am', 'Hillshade_Noon', 'Hillshade_3pm'])) , 
('dd', SmartCorrelatedSelection()),
]
pipeline_4 = Pipeline(pipeline_list_4)
 X_train_pipe_4 = pipeline_4.fit_transform(X_train)
 X_train_pipe_4.head()
from fastai import *
from fastai.vision import *
import pandas as pd<load_from_csv>
pipeline_list_5 = [ 
('dropuniquefeatures', DropIdentifierFeatures()), 
('soil_columns_dummies', FromDummiesToCategories(cols_to_operate=soil_columns, new_column_name='Soil_Type')) , 
('wilder_columns_dummies', FromDummiesToCategories(cols_to_operate=wilder_columns, new_column_name='Wilderness')) , 
('soil_rare', RareLabelEncoder(tol=0.05, variables=['Soil_Type'])) , 
('dp', DropConstantFeatures(tol=0.99)) , 
('dt', DistanceTransformer()), 
('one_hot',OneHotEncoder(variables=['Soil_Type', 'Wilderness'])) , 
('winds', Winsorizer(variables=['EuclideanDistanceHidroloy', 'Elevation',
'Hillshade_9am', 'Hillshade_Noon', 'Hillshade_3pm'])) , 
('dd', SmartCorrelatedSelection()),
]
pipeline_5 = Pipeline(pipeline_list_5)
 X_train_pipe_5 = pipeline_5.fit_transform(X_train)
 X_train_pipe_5.head()
labels = pd.read_csv('.. /input/train.csv')
 def get_labels(name):
return labels[labels['id'] == name.name]['has_cactus'].values[0]<categorify>
pipe_list_1_rf = [('rf', RandomForestClassifier(random_state=42)) ]
pipe_1_rf = Pipeline(pipeline_list_1 + pipe_list_1_rf)
 rfpg1 ={
'rf__n_estimators': [100, 150, 200],
'rf__max_depth': [60,80, None],
'rf__min_samples_split': [2, 3, 4],
}
grid1_pipe_1 = GridSearchCV(pipe_1_rf, param_grid=rfpg1, cv=sss, n_jobs=-1, verbose=3)
 grid1_pipe_1.fit(X_train, y_train)
 print("Best cross-validation accuracy: {:.2f}".format(grid1_pipe_1.best_score_))
print("Test set score: {:.2f}".format(grid1_pipe_1.score(X_test, y_test)))
 print("Best parameters: {}".format(grid1_pipe_1.best_params_))
data = ImageList.from_folder('.. /input/train/train/' ).split_by_rand_pct(0.01 ).label_from_func(get_labels ).transform(get_transforms() , size=224 ).add_test(ItemList.from_folder('.. /input/test/test/')).databunch(bs=64 ).normalize(imagenet_stats )<choose_model_class>
y_predict_pipe_1 = grid1_pipe_1.predict(X_test)
 pd.DataFrame(metrics.confusion_matrix(y_test, y_predict_pipe_1, labels=y_test.unique().tolist()), columns =y_test.unique().tolist() , index = y_test.unique().tolist() )
learner = cnn_learner(data=data, base_arch=models.densenet201, model_dir='.. /.. /.. /.. /models/', metrics=accuracy )<train_model>
print(metrics.classification_report(y_test, y_predict_pipe_1, digits=3))
learner.fit_one_cycle(5, slice(5e-2))<predict_on_test>
pipe_list_2_rf = [('rf', RandomForestClassifier(random_state=42)) ]
pipe_2_rf = Pipeline(pipeline_list_2 + pipe_list_2_rf)
 rfpg2 ={
'rf__n_estimators': [100, 150, 200],
'rf__max_depth': [60,80, None],
'rf__min_samples_split': [2, 3, 4],
}
grid1_pipe_2 = GridSearchCV(pipe_2_rf, param_grid=rfpg2, cv=sss, n_jobs=-1, verbose=3)
 grid1_pipe_2.fit(X_train, y_train)
 print("Best cross-validation accuracy: {:.2f}".format(grid1_pipe_2.best_score_))
print("Test set score: {:.2f}".format(grid1_pipe_2.score(X_test, y_test)))
 print("Best parameters: {}".format(grid1_pipe_2.best_params_))
preds = learner.get_preds(DatasetType.Test )<save_to_csv>
y_predict_pipe_2 = grid1_pipe_2.predict(X_test)
 pd.DataFrame(metrics.confusion_matrix(y_test, y_predict_pipe_2, labels=y_test.unique().tolist()), columns =y_test.unique().tolist() , index = y_test.unique().tolist() )
predictions = preds[0].argmax(dim=1)
 names = [item.name for item in data.test_ds.x.items]
pd.DataFrame({'id': names, 'has_cactus': predictions} ).to_csv('submission.csv', index=False )<train_model>
print(metrics.classification_report(y_test, y_predict_pipe_2, digits=3))
base_dir = ".. /input/"
train_dir = os.path.join(base_dir,"train/train")
 test_dir = os.path.join(base_dir, "test/test")
 print("Training images :
{}".format(os.listdir(train_dir)[:10]), end='
')
 print("Testing images :
{}".format(os.listdir(test_dir)[:10]))
testing_dir = os.path.join(base_dir, "test" )<load_from_csv>
test = pd.read_csv('.. /input/forest-cover-type-prediction/test.csv')
 test.head()
train_dataframe = pd.read_csv(".. /input/train.csv")
 train_dataframe["has_cactus"] = np.where(train_dataframe["has_cactus"] == 1, "yes", "no")
 print(train_dataframe.head() )<import_modules>
test['Cover_Type'] = grid1_pipe_1.predict(test )
import keras
from keras.models import Sequential
from keras.layers import *
from keras.preprocessing.image import ImageDataGenerator
from keras import applications
from efficientnet import EfficientNetB3<prepare_x_and_y>
to_kaggle = test[['Id', 'Cover_Type']].copy()
train_datagen = ImageDataGenerator(
 rescale=1/255,
validation_split=0.10,
rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest' 
)
 shuffeled_df=train_dataframe.sample(frac=1)
 train_generator = train_datagen.flow_from_dataframe(
 dataframe = shuffeled_df[:14000],
directory = train_dir,
x_col="id",
y_col="has_cactus",
target_size=(32,32),
batch_size=250,
shuffle=True,
class_mode="binary" 
)
 valid_datagen = ImageDataGenerator(
 rescale=1/255 
)
 valid_generator = valid_datagen.flow_from_dataframe(
 dataframe = shuffeled_df[14000:],
directory = train_dir,
x_col="id",
y_col="has_cactus",
target_size=(32,32),
batch_size=125,
shuffle=True,
class_mode="binary"
)<choose_model_class>
to_kaggle.to_csv('grid1_pipe_1', index=False )