Kaggle competition: Plant Seedlings Classification
data.columns[data.isnull().sum() >0]<data_type_conversions>
from fastai import *
from fastai.vision import *
import matplotlib.pyplot as plt
def filling_missing_values(data,variable, new_value):
data[variable] = data[variable].fillna(new_value )<load_pretrained>
path = Path('input/train/' )
filling_missing_values(data,'GarageCond','None')
 filling_missing_values(data,'GarageQual','None')
 filling_missing_values(data,'FireplaceQu','None')
 filling_missing_values(data,'BsmtCond','None')
 filling_missing_values(data,'BsmtQual','None')
 filling_missing_values(data,'PoolQC','None')
 filling_missing_values(data,'MiscFeature','None' )<feature_engineering>
np.random.seed(42)
 data = ImageDataBunch.from_folder(path,test='.. /test', ds_tfms=get_transforms() ,valid_pct=0.25,size=299,bs=32,num_workers=0)
 data.normalize(imagenet_stats )
data['MSSubClass'][data['MSSubClass'] == 20] = '1-STORY 1946 & NEWER ALL STYLES'
data['MSSubClass'][data['MSSubClass'] == 30] = '1-STORY 1945 & OLDER'
data['MSSubClass'][data['MSSubClass'] == 40] = '1-STORY W/FINISHED ATTIC ALL AGES'
data['MSSubClass'][data['MSSubClass'] == 45] = '1-1/2 STORY - UNFINISHED ALL AGES'
data['MSSubClass'][data['MSSubClass'] == 50] = '1-1/2 STORY FINISHED ALL AGES'
data['MSSubClass'][data['MSSubClass'] == 60] = '2-STORY 1946 & NEWER'
data['MSSubClass'][data['MSSubClass'] == 70] = '2-STORY 1945 & OLDER'
data['MSSubClass'][data['MSSubClass'] == 75] = '2-1/2 STORY ALL AGES'
data['MSSubClass'][data['MSSubClass'] == 80] = 'SPLIT OR MULTI-LEVEL'
data['MSSubClass'][data['MSSubClass'] == 85] = 'SPLIT FOYER'
data['MSSubClass'][data['MSSubClass'] == 90] = 'DUPLEX - ALL STYLES AND AGES'
data['MSSubClass'][data['MSSubClass'] == 120] = '1-STORY PUD(Planned Unit Development)- 1946 & NEWER'
data['MSSubClass'][data['MSSubClass'] == 150] = '1-1/2 STORY PUD - ALL AGES'
data['MSSubClass'][data['MSSubClass'] == 160] = '2-STORY PUD - 1946 & NEWER'
data['MSSubClass'][data['MSSubClass'] == 180] = 'PUD - MULTILEVEL - INCL SPLIT LEV/FOYER'
data['MSSubClass'][data['MSSubClass'] == 190] = '2 FAMILY CONVERSION - ALL STYLES AND AGES'<feature_engineering>
data.show_batch(rows=3,figsize=(7,6))
def fixing_ordinal_variables(data, variable):
data[variable][data[variable] == 'Ex'] = 5
data[variable][data[variable] == 'Gd'] = 4
data[variable][data[variable] == 'TA'] = 3
data[variable][data[variable] == 'Fa'] = 2
data[variable][data[variable] == 'Po'] = 1
data[variable][data[variable] == 'None'] = 0<drop_column>
learn = create_cnn(data,models.resnet50,metrics=error_rate )
fixing_ordinal_variables(data,'ExterQual')
 fixing_ordinal_variables(data,'ExterCond')
 fixing_ordinal_variables(data,'BsmtCond')
 fixing_ordinal_variables(data,'BsmtQual')
 fixing_ordinal_variables(data,'HeatingQC')
 fixing_ordinal_variables(data,'KitchenQual')
 fixing_ordinal_variables(data,'FireplaceQu')
 fixing_ordinal_variables(data,'GarageQual')
 fixing_ordinal_variables(data,'GarageCond')
 fixing_ordinal_variables(data,'PoolQC' )<feature_engineering>
learn.lr_find()
data['PavedDrive'][data['PavedDrive'] == 'Y'] = 3
data['PavedDrive'][data['PavedDrive'] == 'P'] = 2
data['PavedDrive'][data['PavedDrive'] == 'N'] = 1<count_missing_values>
learn.fit_one_cycle(6,slice(1e-2))
colu = data.columns[(data.isnull().sum() <50)&(data.isnull().sum() >0)]
for i in colu:
print(data[colu].isnull().sum() )<count_missing_values>
learn.save('stg-1' )
colu = data.columns[data.isnull().sum() >=50]
for i in colu:
print(data[colu].isnull().sum() )<categorify>
learn.lr_find()
filling_missing_values(data, 'GarageArea',0)
 filling_missing_values(data, 'GarageCars',0)
 data['GarageFinish'][(data.GarageFinish.isnull() ==True)&(data.GarageCond==0)] =0
data['GarageType'][(data.GarageType.isnull() ==True)&(data.GarageCond==0)] =0
data['GarageYrBlt'][(data.GarageYrBlt.isnull() ==True)&(data.GarageCond==0)] =0<feature_engineering>
learn.unfreeze()
learn.fit_one_cycle(4,max_lr=slice(1e-4,1e-3))
print(data[['MiscFeature','MiscVal']][(data.MiscFeature=='None')&(data.MiscVal>0)])
 data.MiscVal.loc[2550] = 0
print(data[['MiscFeature','MiscVal']][(data.MiscVal==0)&(data.MiscFeature!='None')])
 c=data[['MiscFeature','MiscVal']][(data.MiscVal==0)&(data.MiscFeature!='None')].index
data.MiscFeature.loc[c] = 'None'<train_on_grid>
data = ImageDataBunch.from_folder(path,test='.. /test', ds_tfms=get_transforms() ,valid_pct=0.25,size=350,bs=32,num_workers=0)
 data.normalize(imagenet_stats )
def inputing(variab):
y = data[variab]
data2 = data.drop([variab],axis=1)
 col = data2.columns[data2.isnull().sum() ==0]
data2 = data2[col]
data2 = pd.get_dummies(data2)
 c_train = y[y.notnull() ==True].index
y_train = y[c_train]
columny = data2.columns
X_train = data2[columny].loc[c_train]
c_test = y[y.notnull() !=True].index
y_test = y[c_test]
X_test = data2[columny].loc[c_test]
model = RandomForestClassifier()
model.fit(X_train, y_train)
 y_pred = model.predict(X_test)
 y_pred = pd.Series(y_pred, index=c_test)
 data[variab].loc[c_test] = y_pred.loc[c_test]
def inputingnum(variab):
y = data[variab]
data2 = data.drop([variab],axis=1)
 col = data2.columns[data2.isnull().sum() ==0]
data2 = data2[col]
data2 = pd.get_dummies(data2)
 c_train = y[y.notnull() ==True].index
y_train = y[c_train]
columny = data2.columns
X_train = data2[columny].loc[c_train]
c_test = y[y.notnull() !=True].index
y_test = y[c_test]
X_test = data2[columny].loc[c_test]
model = RandomForestRegressor()
model.fit(X_train, y_train)
 y_pred = model.predict(X_test)
 y_pred = pd.Series(y_pred, index=c_test)
 data[variab].loc[c_test] = y_pred.loc[c_test]<data_type_conversions>
interp = ClassificationInterpretation.from_learner(learn)
 print(metrics.classification_report(interp.y_true.numpy() , interp.pred_class.numpy() ,target_names =data.classes))
inputing(variab='Electrical')
 inputing(variab='Exterior2nd')
 inputing(variab='Exterior1st')
 inputing(variab='MasVnrType')
 inputing(variab='Functional')
 inputing(variab='MSZoning')
 inputing(variab='SaleType')
 inputing(variab='Alley')
 inputing(variab='BsmtExposure')
 inputing(variab='BsmtFinType1')
 inputing(variab='BsmtFinType2')
 inputing(variab='Fence')
 inputingnum(variab='KitchenQual')
 data['KitchenQual'] = data.KitchenQual.astype(int)
 inputingnum(variab='BsmtFullBath')
 data['BsmtFullBath'] = data.BsmtFullBath.astype(int)
 inputingnum(variab='BsmtHalfBath')
 data['BsmtHalfBath'] = data.BsmtHalfBath.astype(int)
 inputingnum(variab='TotalBsmtSF')
 inputingnum(variab='BsmtFinSF1')
 inputingnum(variab='BsmtFinSF2')
 inputingnum(variab='MasVnrArea')
 inputingnum(variab='BsmtUnfSF')
 inputingnum(variab='LotFrontage' )<count_values>
learn.save('stg-2' )
print(data['Utilities'].value_counts())
 data = data.drop(['Utilities'],axis=1 )<filter>
learn.data=data
data.columns[data.isnull().sum() >0]<drop_column>
learn.unfreeze()
learn.fit_one_cycle(4,max_lr=slice(1e-5,1e-4))
def dropping_outliers(data, condition):
condition_to_drop = data[condition].index
data = data.drop(condition_to_drop )<drop_column>
preds,y=learn.get_preds(ds_type=DatasetType.Test )
dropping_outliers(data,(data.SalePrice<100000)&(data.train==1)&(data.LotFrontage>150))
dropping_outliers(data,(data.LotFrontage>200)&(data.train==1))
dropping_outliers(data,(data.SalePrice>700000)&(data.train==1))
dropping_outliers(data,(data.SalePrice>700000)&(data.train==1))
dropping_outliers(data,(data.LotArea>60000)&(data.train==1))
dropping_outliers(data,(data.MasVnrArea>1450)&(data.train==1))
dropping_outliers(data,(data.BedroomAbvGr==8)&(data.train==1))
dropping_outliers(data,(data.KitchenAbvGr==3)&(data.train==1))
dropping_outliers(data,(data['3SsnPorch']>400)&(data.train==1))
dropping_outliers(data,(data.LotArea>100000)&(data.train==1))
dropping_outliers(data,(data.MasVnrArea>1300)&(data.train==1))
dropping_outliers(data,(data.BsmtFinSF1>2000)&(data.train==1)&(data.SalePrice<300000))
dropping_outliers(data,(data.BsmtFinSF2>200)&(data.SalePrice>350000)&(data.train==1))
dropping_outliers(data,(data.BedroomAbvGr==8)&(data.train==1))
dropping_outliers(data,(data.KitchenAbvGr==3)&(data.train==1))
dropping_outliers(data,(data.TotRmsAbvGrd==2)&(data.train==1))<feature_engineering>
preds = np.argmax(preds, axis = 1)
 preds_classes = [data.classes[i] for i in preds]
<feature_engineering><EOS> 
submission = pd.DataFrame({ 'file': os.listdir('input/test'), 'species': preds_classes })
 submission.to_csv('test_classification_results.csv', index=False )