Kaggle competition: Aerial Cactus Identification
<SOS> metric: AUC Kaggle data source: aerial-cactus-identification<count_missing_values>
%matplotlib inline
train_on_gpu = True
print("
print("
print()
print("Train Features = ", train_data.columns.values)
 print("Test Features = ", test_data.columns.values)
 print()
print("NaNs in each training Feature")
 print(train_data.isnull().sum())
 print()
print("NaNs in each testing Feature")
 print(test_data.isnull().sum() )<drop_column>
!pip install albumentations > /dev/null 2>&1
!pip install pretrainedmodels > /dev/null 2>&1
!pip install kekas > /dev/null 2>&1
!pip install adabound > /dev/null 2>&1
train_data.drop(columns='Cabin', inplace=True)
 train_data.drop(columns='Ticket', inplace=True )<define_variables>
import albumentations
from albumentations import torch as AT
import pretrainedmodels
import adabound
from kekas import Keker, DataOwner, DataKek
from kekas.transformations import Transformer, to_torch, normalize
from kekas.metrics import accuracy
from kekas.modules import Flatten, AdaptiveConcatPool2d
from kekas.callbacks import Callback, Callbacks, DebuggerCallback
from kekas.utils import DotDict
def detect_outliers(df,n,features):
outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers<data_type_conversions>
labels = pd.read_csv('.. /input/train.csv')
 fig = plt.figure(figsize=(25, 8))
train_imgs = os.listdir(".. /input/train/train")
 for idx, img in enumerate(np.random.choice(train_imgs, 20)) :
ax = fig.add_subplot(4, 20//4, idx+1, xticks=[], yticks=[])
 im = Image.open(".. /input/train/train/" + img)
 plt.imshow(im)
 lab = labels.loc[labels['id'] == img, 'has_cactus'].values[0]
ax.set_title(f'Label: {lab}' )
train_data['Embarked'].fillna('S', inplace=True )<groupby>
test_img = os.listdir('.. /input/test/test')
 test_df = pd.DataFrame(test_img, columns=['id'])
 test_df['has_cactus'] = -1
test_df['data_type'] = 'test'
labels['has_cactus'] = labels['has_cactus'].astype(int)
 labels['data_type'] = 'train'
labels.head()
meanAges = train_data.groupby(['Pclass'])['Age'].mean()
print(meanAges )<feature_engineering>
labels.loc[labels['data_type'] == 'train', 'has_cactus'].value_counts()
train_data['newGender'] = train_data['Sex']
train_data.loc[train_data['Age']<16., 'newGender'] = 'child'
train_data.drop(columns='Sex', inplace=True)
 print(train_data.head(8))<categorify>
train, valid = train_test_split(labels, stratify=labels.has_cactus, test_size=0.2 )
train_data['Title'] = train_data['Name']
for name_string in train_data['Name']:
train_data['Title'] = train_data['Name'].str.extract('([A-Za-z]+)\.', expand=True)
 mapping = {'Mlle': 'Miss', 'Major': 'Rare', 'Col': 'Rare', 'Sir': 'Rare', 'Don': 'Rare', 'Mme': 'Miss',
'Jonkheer': 'Rare', 'Lady': 'Rare', 'Capt': 'Rare', 'Countess': 'Rare', 'Ms': 'Miss', 'Dona': 'Mrs', 'Rev': 'Rare'}
train_data.replace({'Title': mapping}, inplace=True)
 train_data.drop(columns='Name', inplace=True )<feature_engineering>
def reader_fn(i, row):
image = cv2.imread(f".. /input/{row['data_type']}/{row['data_type']}/{row['id']}")[:,:,::-1]
label = torch.Tensor([row["has_cactus"]])
 return {"image": image, "label": label}
train_data['familyNumber'] = train_data['SibSp'] + train_data['Parch']
print(train_data.head(8))
Outliers_to_drop = detect_outliers(train_data,1,["Age","familyNumber","Fare"])
 train_data = train_data.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True)
 train_data['familySize'] = np.nan
train_data.loc[train_data['familyNumber']==0, 'familySize'] = 'alone'
train_data.loc[train_data['familyNumber']==1, 'familySize'] = 'couple'
train_data.loc[train_data['familyNumber']==2, 'familySize'] = 'small'
train_data.loc[train_data['familyNumber']==3, 'familySize'] = 'medium'
train_data.loc[train_data['familyNumber']>=4, 'familySize'] = 'large'
train_data.drop(columns=['familyNumber', 'SibSp', 'Parch'], inplace=True )<categorify>
def augs(p=0.5):
return albumentations.Compose([
albumentations.HorizontalFlip() ,
albumentations.VerticalFlip() ,
albumentations.RandomBrightness() ,
], p=p )
train_data['newPclass'] = np.nan
train_data.loc[train_data['Pclass']==1, 'newPclass'] = 'first'
train_data.loc[train_data['Pclass']==2, 'newPclass'] = 'second'
train_data.loc[train_data['Pclass']==3, 'newPclass'] = 'third'
train_data.drop(columns='Pclass', inplace=True)
 columns = ['newGender', 'Embarked', 'Title', 'familySize', 'newPclass']
one_hot = pd.get_dummies(train_data.loc[:, columns], drop_first=True)
 train_data.drop(columns=columns, inplace=True)
 train_data = train_data.join(one_hot)
 print(train_data.head(2))
print(train_data.columns.values )<import_modules>
def get_transforms(dataset_key, size, p):
PRE_TFMS = Transformer(dataset_key, lambda x: cv2.resize(x,(size, size)))
 AUGS = Transformer(dataset_key, lambda x: augs()(image=x)["image"])
 NRM_TFMS = transforms.Compose([
Transformer(dataset_key, to_torch()),
Transformer(dataset_key, normalize())
 ])
 train_tfms = transforms.Compose([PRE_TFMS, AUGS, NRM_TFMS])
 val_tfms = transforms.Compose([PRE_TFMS, NRM_TFMS])
 return train_tfms, val_tfms
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import KNeighborsClassifier<prepare_x_and_y>
train_tfms, val_tfms = get_transforms("image", 32, 0.5 )
trainScaler = StandardScaler()
trainScaler.fit(train_data.drop(['Survived', 'PassengerId'], axis = 1))
X_train_scaled = trainScaler.transform(train_data.drop(['Survived', 'PassengerId'], axis = 1))
X_train = train_data.drop(['Survived', 'PassengerId'], axis = 1)
 y_train = train_data['Survived']<train_on_grid>
train_dk = DataKek(df=train, reader_fn=reader_fn, transforms=train_tfms)
 val_dk = DataKek(df=valid, reader_fn=reader_fn, transforms=val_tfms)
 batch_size = 64
workers = 0
train_dl = DataLoader(train_dk, batch_size=batch_size, num_workers=workers, shuffle=True, drop_last=True)
 val_dl = DataLoader(val_dk, batch_size=batch_size, num_workers=workers, shuffle=False )
grid_params = {
'n_neighbors': [3, 5, 7, 9, 11, 13],
'weights': ['uniform', 'distance'],
'metric':['euclidean', 'manhattan']
}
KNN_CV = GridSearchCV(estimator = KNeighborsClassifier() , param_grid=grid_params, cv = 3)
 KNN_CV.fit(X_train_scaled, y_train)
 print(KNN_CV.best_score_)
 weights = KNN_CV.best_params_.get('weights')
 n = KNN_CV.best_params_.get('n_neighbors')
 metric = KNN_CV.best_params_.get('metric')
 bestKNN = KNeighborsClassifier(n_neighbors=n, weights=weights, metric=metric)
 bestKNN.fit(X_train_scaled, y_train )<train_on_grid>
test_dk = DataKek(df=test_df, reader_fn=reader_fn, transforms=val_tfms)
 test_dl = DataLoader(test_dk, batch_size=batch_size, num_workers=workers, shuffle=False )
param_grid = {
'criterion' : ['gini'],
'n_estimators': [70, 80, 90, 100, 110, 120],
'max_features': ['auto', 'log2'],
'max_depth' : [5, 7, 9, 11, 13]
}
randomForest_CV = GridSearchCV(estimator = RandomForestClassifier() , param_grid = param_grid, cv = 3)
 randomForest_CV.fit(X_train, y_train)
 print(randomForest_CV.best_score_)
 criterion = randomForest_CV.best_params_.get('criterion')
 nRF = randomForest_CV.best_params_.get('n_estimators')
 max_features = randomForest_CV.best_params_.get('max_features')
 max_depth = randomForest_CV.best_params_.get('max_depth')
 bestRF = RandomForestClassifier(n_estimators=1000, max_depth=max_depth, max_features=max_features, criterion=criterion)
 bestRF.fit(X_train, y_train )<data_type_conversions>
class Net(nn.Module):
def __init__(
 self,
num_classes: int,
p: float = 0.2,
pooling_size: int = 2,
last_conv_size: int = 1664,
arch: str = "densenet169",
pretrained: str = "imagenet")-> None:

 super().__init__()
net = pretrainedmodels.__dict__[arch](pretrained=pretrained)
 modules = list(net.children())[:-1]
modules += [nn.Sequential(
 Flatten() ,
nn.BatchNorm1d(1664),
nn.Dropout(p),
nn.Linear(1664, num_classes)
)]
self.net = nn.Sequential(*modules)
 def forward(self, x):
logits = self.net(x)
 return logits
for i in range(3):
test_data.loc[test_data['Pclass']==int(i+1), 'Age'] = test_data.loc[test_data['Pclass']==int(i+1), 'Age'].fillna(meanAges.values[i])
 meanFare = test_data['Fare'].median()
test_data['Fare'].fillna(meanFare, inplace=True )<feature_engineering>
dataowner = DataOwner(train_dl, val_dl, None)
 model = Net(num_classes=1)
 criterion = nn.BCEWithLogitsLoss()
test_data['newGender'] = test_data['Sex']
test_data.loc[test_data['Age']<18., 'newGender'] = 'child'
test_data.drop(columns='Sex', inplace=True)
 test_data['Title'] = test_data['Name']
for name_string in test_data['Name']:
test_data['Title'] = test_data['Name'].str.extract('([A-Za-z]+)\.', expand=True)
 mapping = {'Mlle': 'Miss', 'Major': 'Rare', 'Col': 'Rare', 'Sir': 'Rare', 'Don': 'Rare', 'Mme': 'Miss',
'Jonkheer': 'Rare', 'Lady': 'Rare', 'Capt': 'Rare', 'Countess': 'Rare', 'Ms': 'Miss', 'Dona': 'Mrs', 'Rev': 'Rare'}
test_data.replace({'Title': mapping}, inplace=True)
 test_data.drop(columns='Name', inplace=True)
 test_data['familyNumber'] = test_data['SibSp'] + test_data['Parch']
test_data['familySize'] = np.nan
test_data.loc[test_data['familyNumber']==0, 'familySize'] = 'alone'
test_data.loc[test_data['familyNumber']==1, 'familySize'] = 'couple'
test_data.loc[test_data['familyNumber']==2, 'familySize'] = 'small'
test_data.loc[test_data['familyNumber']==3, 'familySize'] = 'medium'
test_data.loc[test_data['familyNumber']>=4, 'familySize'] = 'large'
test_data.drop(columns=['familyNumber', 'SibSp', 'Parch'], inplace=True)
 test_data['newPclass'] = np.nan
test_data.loc[test_data['Pclass']==1, 'newPclass'] = 'first'
test_data.loc[test_data['Pclass']==2, 'newPclass'] = 'second'
test_data.loc[test_data['Pclass']==3, 'newPclass'] = 'third'
test_data.drop(columns='Pclass', inplace=True )<categorify>
def step_fn(model: torch.nn.Module,
batch: torch.Tensor)-> torch.Tensor:

 inp = batch["image"]
return model(inp )
test_data.drop(columns=['Cabin', 'Ticket'], inplace=True)
 one_hot = pd.get_dummies(test_data.loc[:, columns], drop_first=True)
 test_data.drop(columns=columns, inplace=True)
 test_data = test_data.join(one_hot)
 print("NaNs in each testing Feature")
 print(test_data.isnull().sum() )<save_to_csv>
def bce_accuracy(target: torch.Tensor,
preds: torch.Tensor,
thresh: bool = 0.5)-> float:
target = target.cpu().detach().numpy()
preds =(torch.sigmoid(preds ).cpu().detach().numpy() > thresh ).astype(int)
 return accuracy_score(target, preds)
 def roc_auc(target: torch.Tensor,
preds: torch.Tensor)-> float:
target = target.cpu().detach().numpy()
preds = torch.sigmoid(preds ).cpu().detach().numpy()
return roc_auc_score(target, preds )
X_test_scaled = trainScaler.transform(test_data.drop(['PassengerId'], axis = 1))
X_test = test_data.drop(columns='PassengerId')
 testPredictionsKNN = bestKNN.predict(X_test_scaled)
 testPredictionsRF = bestRF.predict(X_test)
 outputKNN = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': testPredictionsKNN})
 outputKNN.to_csv('my_submissionKNN.csv', index=False)
 outputRF = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': testPredictionsRF})
 outputRF.to_csv('my_submissionRF.csv', index=False )<train_on_grid>
keker = Keker(model=model,
dataowner=dataowner,
criterion=criterion,
step_fn=step_fn,
target_key="label",
metrics={"acc": bce_accuracy, 'auc': roc_auc},
opt=torch.optim.SGD,
opt_params={"momentum": 0.99} )
param_grid = {
'learning_rate' : [0.1, 0.2],
'n_estimators': [90, 100, 110],
'loss': ['deviance', 'exponential']
}
gradBoost_CV = GridSearchCV(estimator = GradientBoostingClassifier() , param_grid = param_grid, cv = 3)
 gradBoost_CV.fit(X_train, y_train)
 print(gradBoost_CV.best_score_)
 learning = gradBoost_CV.best_params_.get('learning_rate')
 nGB = gradBoost_CV.best_params_.get('n_estimators')
 loss = gradBoost_CV.best_params_.get('loss')
 bestGB = GradientBoostingClassifier(learning_rate=learning, n_estimators=nGB, loss=loss)
 bestGB.fit(X_train, y_train)
 testPredictionsGB = bestGB.predict(X_test)
 outputGB = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': testPredictionsGB})
 outputGB.to_csv('my_submissionGB.csv', index=False )<train_on_grid>
keker.unfreeze(model_attr="net")
 layer_num = -1
keker.freeze_to(layer_num, model_attr="net" )
param_grid = {
'learning_rate' : [0.1, 0.2],
'max_depth': [3, 5, 7],
}
xgBoost_CV = GridSearchCV(estimator = XGBClassifier() , param_grid = param_grid, cv = 3)
 xgBoost_CV.fit(X_train, y_train)
 print(xgBoost_CV.best_score_)
 learning = xgBoost_CV.best_params_.get('learning_rate')
 depth = xgBoost_CV.best_params_.get('max_depth')
 bestXGB = XGBClassifier(learning_rate=learning, max_depth=depth)
 print(bestXGB)
 bestXGB.fit(X_train, y_train)
 testPredictionsXGB = bestXGB.predict(X_test)
 outputGB = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': testPredictionsXGB})
 outputGB.to_csv('my_submissionXGB.csv', index=False )<load_from_csv>
keker.kek_one_cycle(max_lr=1e-2,
cycle_len=4,
momentum_range=(0.95, 0.85),
div_factor=25,
increase_fraction=0.3,
logdir='train_logs')
 keker.plot_kek('train_logs' )
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_data = pd.read_csv("/kaggle/input/titanic/test.csv")
 train_data.head()<count_missing_values>
keker.kek_one_cycle(max_lr=1e-3,
cycle_len=4,
momentum_range=(0.95, 0.85),
div_factor=25,
increase_fraction=0.2,
logdir='train_logs1')
 keker.plot_kek('train_logs1' )
print("
print("
print()
print("Train Features = ", train_data.columns.values)
 print("Test Features = ", test_data.columns.values)
 print()
print("NaNs in each training Feature")
 print(train_data.isnull().sum())
 print()
print("NaNs in each testing Feature")
 print(test_data.isnull().sum() )<drop_column>
preds = keker.predict_loader(loader=test_dl )
train_data.drop(columns='Cabin', inplace=True)
 train_data.drop(columns='Ticket', inplace=True )<define_variables>
<data_type_conversions><EOS> 
test_preds = pd.DataFrame({'imgs': test_df.id.values, 'preds': preds.reshape(-1,)})
 test_preds.columns = ['id', 'has_cactus']
test_preds.to_csv('sub.csv', index=False)
 test_preds.head()