Kaggle competition: Planet: Understanding the Amazon from Space
<SOS> metric: MeanFScoreBeta Kaggle data source: planet:-understanding-the-amazon-from-space<load_from_csv>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
ds_train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 ds_test = pd.read_csv('.. /input/digit-recognizer/test.csv')
<prepare_x_and_y>
!pip install fastai==0.7.0 --no-deps
!pip install torch==0.4.1 torchvision==0.2.1
val_size = 0.1
X_all = ds_train.loc[:, ds_train.columns != 'label'].to_numpy()
y_all = ds_train['label'].to_numpy()
test = ds_test.to_numpy()<split>
print(os.listdir(".. /input/"))
PATH = '.. /input/planet-understanding-the-amazon-from-space/'
X_all = X_all / 255.0
X_all = X_all.reshape(-1,28,28,1)
 test = test / 255.0
test = test.reshape(-1,28,28,1)
 y_all = to_categorical(y_all, num_classes = 10)
 X_train, X_val, y_train, y_val = model_selection.train_test_split(X_all, y_all, test_size=val_size, random_state=1 )<choose_model_class>
TMP_PATH = "/tmp/tmp"
MODEL_PATH = "/tmp/model"
optimizer = RMSprop(lr=0.001, rho=0.9, decay=0.0 )<choose_model_class>
df = pd.read_csv(PATH+'sample_submission_v2.csv')
 df.head(5 )
def get_simple_model() :
model = Sequential()
model.add(Flatten(input_shape=(28,28,1)))
 model.add(Dense(64, activation = 'sigmoid'))
model.add(Dropout(0.2))
model.add(Dense(24, activation = 'sigmoid'))
model.add(Dropout(0.2))
model.add(Dense(10, activation = 'softmax'))
model.compile(optimizer = optimizer, loss = "categorical_crossentropy", metrics=["accuracy"])
 return model<choose_model_class>
from fastai.plots import *
def get_cnn_model() :
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))
model.compile(optimizer = optimizer, loss = "categorical_crossentropy", metrics=["accuracy"])
 return model<save_to_csv>
def f2(preds, targs, start=0.17, end=0.24, step=0.01):
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 return max([fbeta_score(targs,(preds>th), 2, average='samples')
 for th in np.arange(start,end,step)])
 metrics=[f2]
f_model = resnet34
def test_submission(model):
results = model.predict(test)
 results = np.argmax(results, axis=1)
 results = pd.Series(results, name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )<train_model>
label_csv = f'{PATH}train_v2.csv'
n = len(list(open(label_csv)))-1
val_idxs = get_cv_idxs(n )
cnn_model_2, _ = train_model(get_cnn_model() , epochs = 30)
 test_submission(cnn_model_2 )<set_options>
def get_data(sz):
tfms = tfms_from_model(f_model, sz, aug_tfms=transforms_top_down, max_zoom=1.05)
 return ImageClassifierData.from_csv(PATH, 'train-jpg', label_csv, tfms=tfms,
suffix='.jpg', val_idxs=val_idxs, test_name='test-jpg-v2' )
warnings.filterwarnings('ignore' )<import_modules>
data = get_data(256 )
import seaborn as sns
import matplotlib.pyplot as plt
from scipy.stats import norm
import scipy.stats<load_from_csv>
x,y = next(iter(data.val_dl))
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )<normalization>
list(zip(data.classes, y[0]))
tri_df=corrmat.mask(mask)
 print([c for c in tri_df.columns if any(tri_df[c]>0.65)] )<define_variables>
sz=64
columns_to_work_engg = ['GarageYrBlt','YearRemodAdd','1stFlrSF','2ndFlrSF','BedroomAbvGr','GarageCars']<filter>
data = get_data(sz )
train_1=train[train['GrLivArea']<4500]<filter>
data = data.resize(int(sz*1.3), TMP_PATH )
train_1=train[train['TotalBsmtSF']<3100]<filter>
learn = ConvLearner.pretrained(f_model, data, metrics=metrics )
train_1=train[train['GarageArea']<1200]<define_variables>
lr = 0.2
num_col=[c for c in train_1.columns if train_1[c].dtype !='object']<feature_engineering>
learn.fit(lr, 3, cycle_len=1, cycle_mult=2 )
skew_before=[]
skew_after=[]
for c in num_col:
skew_before.append(train_1[c].skew())
 skew_after.append(np.log1p(train_1[c] ).skew())
 skew_df=pd.DataFrame(zip(skew_before,skew_after), columns=['skew_before','skew_after'],index=num_col)
 skew_df = skew_df[abs(skew_df['skew_before'])>abs(skew_df['skew_after'])]
skew_df['skew_diff'] = abs(skew_df['skew_before'])-abs(skew_df['skew_after'])
 skew_df[skew_df['skew_diff']>1]<import_modules>
lrs = np.array([lr/9,lr/3,lr] )
from sklearn.preprocessing import PowerTransformer
from scipy.stats import skew<feature_engineering>
learn.unfreeze()
learn.fit(lrs, 3, cycle_len=1, cycle_mult=2 )
log=PowerTransformer()
skew_before=[]
skew_after=[]
for c in num_col:
skew_before.append(train_1[c].skew())
 log.fit(train_1[c].values.reshape(-1,1))
skew_after.append(skew(( log.transform(train_1[c].values.reshape(-1,1)))) [0])
 skew_df=pd.DataFrame(zip(skew_before,skew_after), columns=['skew_before','skew_after'],index=num_col)
 skew_df = skew_df[abs(skew_df['skew_before'])>abs(skew_df['skew_after'])]
skew_df['skew_diff'] = abs(skew_df['skew_before'])-abs(skew_df['skew_after'])
 skew_df[skew_df['skew_diff']>1]<define_variables>
learn.save(f'{sz}' )
columns_to_work_log=['LotArea', 'BsmtFinSF2', 'LowQualFinSF', 'GrLivArea', 'KitchenAbvGr', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'MiscVal']<define_variables>
sz=128
columns_to_work_drop=['MiscVal']<define_variables>
learn.set_data(get_data(sz))
learn.freeze()
learn.fit(lr, 3, cycle_len=1, cycle_mult=2 )
num_cols=[c for c in train_1.columns if train_1[c].dtype !='object']
print(num_cols )<define_variables>
learn.unfreeze()
learn.fit(lrs, 3, cycle_len=1, cycle_mult=2)
 learn.save(f'{sz}' )
columns_to_work_preprocess = ['MSSubClass']<define_variables>
sz=256
cat_cols=[c for c in train_1.columns if train_1[c].dtype =='object']
print(num_cols )<prepare_x_and_y>
learn.set_data(get_data(sz))
learn.freeze()
learn.fit(lr, 3, cycle_len=1, cycle_mult=2 )
y = train_1['SalePrice'].reset_index(drop=True)
 feature_train = train_1.drop('SalePrice',axis=1)
 print(feature_train.shape,y.shape,test.shape )<concatenate>
learn.unfreeze()
learn.fit(lrs, 3, cycle_len=1, cycle_mult=2)
 learn.save(f'{sz}' )
all_data = pd.concat([feature_train,test],ignore_index=True ).reset_index(drop=True)
 all_data.shape<data_type_conversions>
multi_preds, y = learn.TTA()
preds = np.mean(multi_preds, 0 )
all_data['MSSubClass'] = all_data['MSSubClass'].astype('object')
 all_data.MSSubClass.dtype<count_missing_values>
f2(preds,y )
all_data.isna().sum().nlargest(15 )<concatenate>
multi_preds, y = learn.TTA(is_test=True )
<drop_column><EOS> 
test_fnames = [os.path.basename(f ).split(".")[0] for f in data.test_ds.fnames]
classes = np.array(data.classes, dtype=str)
 res = [" ".join(classes[np.where(pp > 0.2)])for pp in preds]
test_df = pd.DataFrame(res, index=test_fnames, columns=['tags'])
 test_df.head(5)
 test_df.to_csv('submission.csv', index_label='image_name')
 df = pd.read_csv('submission.csv')
 df.head(20 )