Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<train_model>
init_notebook_mode(connected=True )
model.fit(X_train, Y_train, epochs=20, batch_size=128,validation_data=(X_valid,Y_valid))<predict_on_test>
print("Loading data files...")
 start = time()
posc_bal = reduce_mem_usage(pd.read_csv(".. /input/POS_CASH_balance.csv"))
bureau_bal = reduce_mem_usage(pd.read_csv(".. /input/bureau_balance.csv"))
app_train = reduce_mem_usage(pd.read_csv(".. /input/application_train.csv"))
prev_app = reduce_mem_usage(pd.read_csv(".. /input/previous_application.csv"))
inst_pay = reduce_mem_usage(pd.read_csv(".. /input/installments_payments.csv"))
cc_bal = reduce_mem_usage(pd.read_csv(".. /input/credit_card_balance.csv"))
app_test = reduce_mem_usage(pd.read_csv(".. /input/application_test.csv"))
bureau = reduce_mem_usage(pd.read_csv(".. /input/bureau.csv"))
end = time()
print("Finished loading data files and running memory optimization in {} seconds.".format(int(round(end - start))))
model.predict(X_valid )<compute_test_metric>
eda = pd.DataFrame(
 [
['Point of Sale Cash Balance', posc_bal.shape[0], posc_bal.shape[1] - 2, np.sum(posc_bal.dtypes=='category'),
np.sum(posc_bal.isnull().sum() > 0), posc_bal.isnull().sum().sum() ],
['Bureau Balance', bureau_bal.shape[0], bureau_bal.shape[1] - 1, np.sum(bureau_bal.dtypes=='category'),
np.sum(bureau_bal.isnull().sum() > 0), bureau_bal.isnull().sum().sum() ],
['Applications Train', app_train.shape[0], app_train.shape[1] - 2, np.sum(app_train.dtypes=='category'),
np.sum(app_train.isnull().sum() > 0), app_train.isnull().sum().sum() ],
['Previous Applications', prev_app.shape[0], prev_app.shape[1] - 2, np.sum(prev_app.dtypes=='category'),
np.sum(prev_app.isnull().sum() > 0), prev_app.isnull().sum().sum() ],
['Installment Payments', inst_pay.shape[0], inst_pay.shape[1] - 2, np.sum(inst_pay.dtypes=='category'),
np.sum(inst_pay.isnull().sum() > 0), inst_pay.isnull().sum().sum() ],
['Credit Card Balance', cc_bal.shape[0], cc_bal.shape[1] - 2, np.sum(cc_bal.dtypes=='category'),
np.sum(cc_bal.isnull().sum() > 0), cc_bal.isnull().sum().sum() ],
['Applications Test', app_test.shape[0], app_test.shape[1] - 1, np.sum(app_test.dtypes=='category'),
np.sum(app_test.isnull().sum() > 0), app_test.isnull().sum().sum() ],
['Bureau', bureau.shape[0], bureau.shape[1] - 2, np.sum(bureau.dtypes=='category'),
np.sum(bureau.isnull().sum() > 0), bureau.isnull().sum().sum() ],
],
columns=['Dataset', 'samples', 'number_features', 'number_categorical_features', 'number_features_missing_values',
'total_number_missing_values'] 
)
 display(eda.head(8))
model.evaluate(X_valid,Y_valid )<compute_test_metric>
app_train['is_train'] = 1
app_train['is_test'] = 0
app_test['is_train'] = 0
app_test['is_test'] = 1
print("
Joining the training(app_train)and testing(app_test)dataset for pre-processing into pandas DataFrame 'data'.")
 data = pd.concat([app_train, app_test], axis=0)
 print("app_train has {0:,} samples and {1} features.".format(app_train.shape[0], app_train.shape[1]-4))
print("app_test has {0:,} samples and {1} features.".format(app_test.shape[0], app_test.shape[1]-3))
print("data has {0:,} samples and {1} features BEFORE one-hot encoding.".format(data.shape[0], data.shape[1]-4))
assert(data.shape[0] == app_train.shape[0] + app_test.shape[0])
 assert(data.shape[1] >= max(app_train.shape[1], app_test.shape[1]))
scores=model.evaluate(test_x,test_y )<compute_test_metric>
def _one_hot_encoding(data):
return pd.get_dummies(data)
 print("
Performing one-hot encoding on {} dataset.".format('data'))
data = _one_hot_encoding(data)
 print("app has {0:,} samples and {1} features AFTER one-hot encoding.".format(data.shape[0], data.shape[1]-4))
posc_bal = _one_hot_encoding(posc_bal)
 prev_app = _one_hot_encoding(prev_app)
 inst_pay = _one_hot_encoding(inst_pay)
 cc_bal = _one_hot_encoding(cc_bal)
 bureau = _one_hot_encoding(bureau )
print(f"Accuracy is {scores[1]*100} %" )<predict_on_test>
data_train_test = data.copy()
output=np.argmax(model.predict(test_x),axis=1 )<create_dataframe>
print("Merge 'Point of Sale Cash Balance' dataset.")
 posc_bal_count = posc_bal[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
posc_bal['POSC_BAL_COUNT'] = posc_bal['SK_ID_CURR'].map(posc_bal_count['SK_ID_PREV'])
 posc_bal = posc_bal.drop(['SK_ID_PREV'], axis=1)
 posc_bal_avg = posc_bal.groupby('SK_ID_CURR' ).mean()
posc_bal_avg.columns = ['pcb_' + col for col in posc_bal_avg.columns]
data = data.merge(right=posc_bal_avg.reset_index() , how='left', on='SK_ID_CURR' )
submission=pd.DataFrame()
submission['name']=test_df['Images'].str.split("/", n = 6, expand = True)[5].str.split(".", n = 3, expand = True)[0]
submission['class']=output<categorify>
submission['class'] = submission['class'].replace({0:1, 1:0} )<save_to_csv>
print("Merge 'Previous Applications' dataset.")
 prev_app_count = prev_app[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
prev_app['PREV_COUNT'] = prev_app['SK_ID_CURR'].map(prev_app_count['SK_ID_PREV'])
 prev_app = prev_app.drop(['SK_ID_PREV'], axis=1)
 prev_app_avg = prev_app.groupby('SK_ID_CURR' ).mean()
prev_app_avg.columns = ['pa_' + col for col in prev_app_avg.columns]
data = data.merge(right=prev_app_avg.reset_index() , how='left', on='SK_ID_CURR' )
submission.to_csv("submission.csv", index=False )<save_to_csv>
print("Merge 'Installments Payments' dataset.")
 inst_pay_count = inst_pay[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
inst_pay['INST_PAY_COUNT'] = inst_pay['SK_ID_CURR'].map(inst_pay_count['SK_ID_PREV'])
 inst_pay = inst_pay.drop(['SK_ID_PREV'], axis=1)
 inst_pay_avg = inst_pay.groupby('SK_ID_CURR' ).mean()
inst_pay_avg.columns = ['ip_' + col for col in inst_pay_avg.columns]
data = data.merge(right=inst_pay_avg.reset_index() , how='left', on='SK_ID_CURR' )
submission.to_csv("submission.csv", index=False )<install_modules>
print("Merge 'Credit Card Balance' dataset.")
 cc_bal_count = cc_bal[['SK_ID_CURR', 'SK_ID_PREV']].groupby('SK_ID_CURR' ).count()
cc_bal['CC_BAL_COUNT'] = cc_bal['SK_ID_CURR'].map(cc_bal_count['SK_ID_PREV'])
 cc_bal = cc_bal.drop(['SK_ID_PREV'], axis=1)
 cc_bal_avg = cc_bal.groupby('SK_ID_CURR' ).mean()
cc_bal_avg.columns = ['ccb_' + col for col in cc_bal_avg.columns]
data = data.merge(right=cc_bal_avg.reset_index() , how='left', on='SK_ID_CURR' )
!pip install torchsummary<set_options>
print("Merge 'Bureau' dataset.")
 bureau_count = bureau[['SK_ID_CURR', 'SK_ID_BUREAU']].groupby('SK_ID_CURR' ).count()
bureau['BUREAU_COUNT'] = bureau['SK_ID_CURR'].map(bureau_count['SK_ID_BUREAU'])
 bureau = bureau.drop(['SK_ID_BUREAU'], axis=1)
 bureau_avg = bureau.groupby('SK_ID_CURR' ).mean()
bureau_avg.columns = ['b_' + col for col in bureau_avg.columns]
data = data.merge(right=bureau_avg.reset_index() , how='left', on='SK_ID_CURR' )
%matplotlib inline<set_options>
def seed_everything(seed=42):
os.environ['PYTHONHASHSEED'] = str(seed)
 random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def get_logger(
 filename='log',
disable_stream_handler=False,
disable_file_handler=False 
):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 if not disable_stream_handler:
handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 if not disable_file_handler:
handler2 = FileHandler(filename=f"{filename}.log")
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler2)
 return logger<set_options>
scaler = MinMaxScaler()
numerical = ['DAYS_BIRTH', 'AMT_CREDIT', 'DAYS_ID_PUBLISH', 'DAYS_REGISTRATION', 'DAYS_EMPLOYED']
data[numerical] = scaler.fit_transform(data[numerical] )
logger = get_logger(
 filename='running',
disable_stream_handler=False,
disable_file_handler=False,
)<define_variables>
data_to_use = 'ALL'
if data_to_use == 'data_train_test':
data = data_train_test.copy()
INPUT_DIR = '.. /input/ailab-ml-training-1/'
ARTIFACT_DIR = '.. /input/v1-ailab1-cv/'
PATH = {
'train': os.path.join(INPUT_DIR, 'train.csv'),
'submission': os.path.join(INPUT_DIR, 'sample_submission.csv'),
'train_image_dir': os.path.join(INPUT_DIR, 'train_images/train_images'),
'test_image_dir': os.path.join(INPUT_DIR, 'test_images/test_images'),
'state_dict': os.path.join(ARTIFACT_DIR, 'best_state_dicts.pth'),
'oof': os.path.join(ARTIFACT_DIR, 'oof.npy'),
'predictions': os.path.join(ARTIFACT_DIR, 'predictions.npy'),
}
ID = 'fname'
TARGET = 'label'
SEED = 42
DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
seed_everything(SEED )<load_from_csv>
print("
Filling NaN values in the dataset using pandas.fillna() using the column mean() value.")
 print("Number of NaN values in the dataset BEFORE running pandas.fillna() : {:,}".format(data.isnull().sum().sum()))
data = data.fillna(data.mean())
 nan_after = data.isnull().sum().sum()
print("Number of NaN values in the dataset AFTER running pandas.fillna() : {:,}".format(nan_after))
assert(nan_after == 0 )
train_df = pd.read_csv(PATH['train'])
 submission_df = pd.read_csv(PATH['submission'] )<feature_engineering>
del posc_bal, posc_bal_count, posc_bal_avg, bureau_bal, app_train, app_test
del prev_app, prev_app_count, prev_app_avg, inst_pay, inst_pay_count, inst_pay_avg, cc_bal, cc_bal_count, cc_bal_avg
del bureau, bureau_count, bureau_avg, data_train_test
gc.collect()
train_df[ID] = train_df[ID].apply(lambda x: os.path.join(PATH['train_image_dir'], x))<prepare_output>
print("
Separating the training and testing dataset after completing pre-processing.")
 train = data[data['is_train'] == 1]
target = train['TARGET']
train = train.drop(['TARGET', 'SK_ID_CURR', 'is_test', 'is_train'], axis=1)
 test = data[data['is_test'] == 1]
test_id = test['SK_ID_CURR']
test = test.drop(['TARGET', 'SK_ID_CURR', 'is_test', 'is_train'], axis=1)
 print("train has {:,} samples and {} features.".format(train.shape[0], train.shape[1]))
print("test has {:,} samples and {} features.".format(test.shape[0], test.shape[1]))
def softmax(logits):
return np.exp(logits)/ np.sum(np.exp(logits), axis=1, keepdims=True )<feature_engineering>
print("
Splitting the training dataset into actual training and validation datasets")
 X_train, X_val, y_train, y_val = train_test_split(train, target, test_size=0.2, random_state=42)
 assert(train.shape[0] == X_train.shape[0] + X_val.shape[0])
 assert(X_train.shape[1] == train.shape[1])
 assert(X_val.shape[1] == train.shape[1])
 assert(target.shape[0] == y_train.shape[0] + y_val.shape[0])
 print("training dataset has {0:,} samples and {1} features.".format(X_train.shape[0], X_train.shape[1]))
print("validating dataset has {0:,} samples and {1} features.".format(X_val.shape[0], X_val.shape[1]))
predictions = np.load(PATH['predictions'])
 predictions = softmax(predictions)
 predictions_label, predictions_proba = np.argmax(predictions, axis=1), np.max(predictions, axis=1)
 pseudo_df = submission_df.copy()
pseudo_df[TARGET] = predictions_label
pseudo_df['proba'] = predictions_proba
pseudo_df = pseudo_df.loc[pseudo_df['proba'] > 0.999, :]
pseudo_df = pseudo_df.drop('proba', axis=1)
 pseudo_df[ID] = pseudo_df[ID].apply(lambda x: os.path.join(PATH['test_image_dir'], x))<count_unique_values>
run_mode = 'LGBM_KFold'
<train_model>
if run_mode == 'grid_search_LGBM':
perc_samples = 0.15
print("
Preparing to run Hyperparameters tunning with GridSearchCV using {0:.2f}% of the training samples".format(perc_samples * 100))
X_train_small = X_train[:int(perc_samples * X_train.shape[0])]
y_train_small = y_train[:int(perc_samples * y_train.shape[0])]
X_val_small = X_val[:int(perc_samples * X_val.shape[0])]
y_val_small = y_val[:int(perc_samples * y_val.shape[0])]
estimator = lgb.LGBMClassifier(
 objective='binary',
metric='auc',
num_iteration=5000,
verbose=1,
silent=False,
colsample_bytree=.8,
subsample=.9,
reg_alpha=.1,
reg_lambda=.1,
min_split_gain=.01,
min_child_weight=1, 
)

parameters = {
'boosting_type': ['gbdt'],
'num_leaves': [35, 48, 80],
'min_data_in_leaf': [20],
'learning_rate': [0.005],
'max_depth': [7],
}
scorer = make_scorer(roc_auc_score)
 grid_obj = GridSearchCV(estimator=estimator,
param_grid=parameters,
scoring=scorer)
 start = time()
grid_fit = grid_obj.fit(X_train_small, y_train_small)
 end = time()
grid_fit_time =(end - start)/ 60
print("
GridSearchCV estimator fit time: {0:.2f} minutes".format(( end - start)/ 60))
print("
Preparing to run Hyperparameters tunning with GridSearchCV using {0:.2f}% of the training samples".format(perc_samples * 100))
print("
Parameters used for tunning:
{}".format(parameters))
best_est = grid_obj.best_estimator_
print("
Best Estimator:
{}
".format(best_est))
best_score = grid_obj.best_score_
print("
Best Estimator Score: {}
".format(best_score))
best_params = grid_obj.best_params_
print("
Best Hyperparameters that yield the best score:
{}
".format(best_params))
Unoptimized Estimator prediction score on Validation set: \t{}".format(roc_auc_score(y_val_small, pred_val)))
 best_pred_val = best_est.predict(X_val_small)
 print("
Optimized Estimator prediction score on Validation set: \t{}".format(roc_auc_score(y_val_small, best_pred_val)))
class KmnistDataset(Dataset):
def __init__(
 self,
paths,
labels,
transform=None,
with_memory_cache=False, 
):
super().__init__()
self.paths = paths
self.labels = labels
self.transform = transform
self.with_memory_cache = with_memory_cache
if with_memory_cache:
self.images = [None,] * len(paths)
 def load_image(self, path):
image = cv2.imread(path)
 return image
def __len__(self):
return len(self.paths)
 def __getitem__(self, idx):
if self.with_memory_cache:
image = self.images[idx]
if image is None:
path = self.paths[idx]
image = self.load_image(path)
 self.images[idx] = image
else:
path = self.paths[idx]
image = self.load_image(path)
 label = self.labels[idx]
if self.transform is not None:
image = self.transform(image=image)['image']
return image, label<create_dataframe>
if run_mode == 'grid_search_RFR':
perc_samples = 0.15
print("
Preparing to run Hyperparameters tunning with GridSearchCV using {0:.2f}% of the training samples".format(perc_samples * 100))
features_train_small = X_train[:int(perc_samples * X_train.shape[0])]
target_train_small = y_train[:int(perc_samples * y_train.shape[0])]
features_val_small = X_val[:int(perc_samples * X_val.shape[0])]
target_val_small = y_val[:int(perc_samples * y_val.shape[0])]
estimator = RandomForestRegressor(n_jobs=-1,
random_state=42,
verbose=0)

parameters = {
'n_estimators': [130, 135, 145],
'min_samples_leaf': [55, 62, 75],
'max_features': [0.2],
'min_samples_split': [2],
}
scorer = make_scorer(roc_auc_score)
 grid_obj = GridSearchCV(estimator=estimator,
param_grid=parameters,
scoring=scorer)
 start = time()
grid_fit = grid_obj.fit(features_train_small, target_train_small)
 end = time()
grid_fit_time =(end - start)/ 60
print("
GridSearchCV estimator fit time: {0:.2f} minutes".format(( end - start)/ 60))
best_est = grid_obj.best_estimator_
print("
Best Estimator:
{}
".format(best_est))
best_score = grid_obj.best_score_
print("
Best Estimator Score: {}
".format(best_score))
best_params = grid_obj.best_params_
print("
Best Hyperparameters that yield the best score:
{}
".format(best_params))
pred_val =(estimator.fit(features_train_small, target_train_small)).predict(features_val_small)
 print("
Unoptimized Estimator prediction score on Validation set: \t{}".format(roc_auc_score(target_val_small, pred_val)))
 best_pred_val = best_est.predict(features_val_small)
 print("
Optimized Estimator prediction score on Validation set: \t{}".format(roc_auc_score(target_val_small, best_pred_val)))
def get_dataloader(
 X,
Y,
transform=None,
with_memory_cache=False,
batch_size=32,
shuffle=False,
num_workers=0,
pin_memory=True, 
):
dataset = KmnistDataset(
 X,
Y,
transform=transform,
with_memory_cache=with_memory_cache, 
)
 loader = DataLoader(
 dataset=dataset,
batch_size=batch_size,
shuffle=shuffle,
num_workers=num_workers,
pin_memory=pin_memory, 
)
 return loader<groupby>
if run_mode == 'train_estimator_LGBM':
params = {
'boosting_type': 'dart',
'objective': 'binary',
'learning_rate': 0.1,
'min_data_in_leaf': 30,
'num_leaves': 31,
'max_depth': -1,
'feature_fraction': 0.5,
'scale_pos_weight': 2,
'drop_rate': 0.02,
'metric': 'auc',
'num_boost_round': 200,
}
data_split = 'kfold'
if data_split == 'train_test_split':
lgb_train = lgb.Dataset(data=X_train, label=y_train)
 lgb_eval = lgb.Dataset(data=X_val, label=y_val)
 start = time()
estimator = lgb.train(
 params = params,
train_set = lgb_train,
valid_sets = lgb_eval,
early_stopping_rounds = 350,
verbose_eval = 200 
)
 end = time()
elif data_split == 'kfold':
lgb_train_cv = lgb.Dataset(data=train, label=target)
 start = time()
cv_results = lgb.cv(
 params = params,
train_set = lgb_train_cv,
nfold = 2,
early_stopping_rounds = 50,
stratified = True,
verbose_eval = 50 
)
 optimum_boost_rounds = np.argmax(cv_results['auc-mean'])
 print('Optimum boost rounds = {}'.format(optimum_boost_rounds))
print('Best LGBM CV result = {}'.format(np.max(cv_results['auc-mean'])))
 estimator = lgb.train(
 params = params,
train_set = lgb_train_cv,
num_boost_round = optimum_boost_rounds,
verbose_eval = 50 
)
 end = time()
print("
Estimator fit time: {} seconds".format(int(round(end - start))))
lgb.plot_importance(estimator, figsize=(12, 12), max_num_features=30);
num_samples = 20
sample = train_df.groupby(TARGET ).apply(lambda df: df.sample(num_samples))
fnames = sample[ID].to_list()
labels = sample[TARGET].to_list()<concatenate>
print(estimator )
def conv3x3(in_planes, out_planes, stride=1, padding=1):
return nn.Conv2d(in_planes, out_planes, kernel_size=3,
stride=stride, padding=padding, bias=False)
 def conv1x1(in_planes, out_planes, stride=1):
return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
 class SSELayer(nn.Module):
def __init__(self, channel):
super().__init__()
self.conv = nn.Sequential(
 conv1x1(channel, 1),
nn.Sigmoid() 
)
 def forward(self, x):
y = self.conv(x)
 return x * y.expand_as(x)
 class CSELayer(nn.Module):
def __init__(self, channel, reduction=16):
super().__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc = nn.Sequential(
 nn.Linear(channel, channel // reduction, bias=False),
nn.ReLU(inplace=True),
nn.Linear(channel // reduction, channel, bias=False),
nn.Sigmoid() 
)
 def forward(self, x):
b, c, _, _ = x.size()
y = self.avg_pool(x ).view(b, c)
 y = self.fc(y ).view(b, c, 1, 1)
 return x * y.expand_as(x)
 class SCSELayer(nn.Module):
def __init__(self, channel, reduction=16):
super().__init__()
self.sse = SSELayer(channel)
 self.cse = CSELayer(channel, reduction)
 def forward(self, x):
return self.sse(x)+ self.cse(x)
 def se_layer(channel, se_type=None, reduction=16):
if se_type is None:
return nn.Identity()
elif se_type == 'cse':
return CSELayer(channel, reduction)
 elif se_type == 'sse':
return SSELayer(channel)
 elif se_type == 'scse':
return SCSELayer(channel, reduction)
 else:
raise NotImplementedError
class BasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None,
norm_layer=None, se_type=None, reduction=16):
super().__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
self.conv1 = conv3x3(inplanes, planes, stride)
 self.bn1 = norm_layer(planes)
 self.relu = nn.ReLU(inplace=True)
 self.conv2 = conv3x3(planes, planes)
 self.bn2 = norm_layer(planes)
 self.se = se_layer(planes, se_type=se_type, reduction=reduction)
 self.downsample = downsample
def forward(self, x):
identity = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.se(out)
 if self.downsample is not None:
identity = self.downsample(x)
 out += identity
out = self.relu(out)
 return out
class ResNet(nn.Module):
def __init__(self, block, layers, num_classes=10, width_per_group=64,
se_type=None, reduction=16):
super(ResNet, self ).__init__()
norm_layer = nn.BatchNorm2d
self._norm_layer = norm_layer
self.inplanes = 64
self.base_width = width_per_group
self.se_type = se_type
self.reduction = reduction
self.conv1 = nn.Conv2d(1, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False)
 self.bn1 = norm_layer(self.inplanes)
 self.relu = nn.ReLU(inplace=True)
 self.layer1 = self._make_layer(block, 64, layers[0])
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
 self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
 self.avgpool = nn.AdaptiveAvgPool2d(( 1, 1))
self.fc = nn.Linear(512 * block.expansion, num_classes)
 for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
 elif isinstance(m,(nn.BatchNorm2d, nn.GroupNorm)) :
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 def _make_layer(self, block, planes, blocks, stride=1):
norm_layer = self._norm_layer
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 conv1x1(self.inplanes, planes * block.expansion, stride),
norm_layer(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, stride, downsample=downsample, norm_layer=norm_layer,
se_type=self.se_type, reduction=self.reduction))
self.inplanes = planes * block.expansion
for _ in range(1, blocks):
layers.append(block(self.inplanes, planes, norm_layer=norm_layer,
se_type=self.se_type, reduction=self.reduction))
return nn.Sequential(*layers)
 def _forward_impl(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.avgpool(x)
 x = torch.flatten(x, 1)
 x = self.fc(x)
 return x
def forward(self, x):
return self._forward_impl(x)
 def small_resnet18(se_type='scse'):
return ResNet(BasicBlock, [2, 2, 2, 2], se_type=se_type)
 def small_resnet34(se_type='scse'):
return ResNet(BasicBlock, [3, 4, 6, 3], se_type=se_type )<compute_train_metric>
if run_mode == 'LGBM_KFold':
folds = KFold(n_splits=10, shuffle=True, random_state=1024)
 oof_preds = np.zeros(train.shape[0])
 sub_preds = np.zeros(test.shape[0])
 feature_importance = pd.DataFrame()
feats = train.columns
start = time()
for n_fold,(train_index, valid_index)in enumerate(folds.split(train, target)) :
train_x, train_y = train.iloc[train_index], target.iloc[train_index]
valid_x, valid_y = train.iloc[valid_index], target.iloc[valid_index]
clf = lgb.LGBMClassifier(
 nthread=4,
n_estimators=10000,
learning_rate=0.02,
num_leaves=34,
colsample_bytree=0.9497036,
subsample=0.8715623,
max_depth=8,
reg_alpha=0.041545473,
reg_lambda=0.0735294,
min_split_gain=0.0222415,
min_child_weight=39.3259775,
silent=-1,
verbose=-1, 
)
 clf.fit(
 train_x,
train_y,
eval_set = [(valid_x, valid_y)],
eval_metric = 'auc',
verbose = 200,
early_stopping_rounds = 500, 
)
 oof_preds[valid_index] = clf.predict_proba(valid_x, num_iterations=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test, num_iterations=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance = pd.DataFrame()
fold_importance['feature'] = feats
fold_importance['importance'] = clf.feature_importances_
fold_importance['fold'] = n_fold + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 print('Fold {:02d} AUC: {:.6f}'.format(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_index])))
 end = time()
print("
Estimator fit time: {} seconds".format(int(round(end - start))))
print('Full AUC score: {:.6f}'.format(roc_auc_score(target, oof_preds)) )
class AverageMeter:
def __init__(self):
self.reset()
def reset(self):
self.val = 0.0
self.avg = 0.0
self.sum = 0.0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def accuracy_score_torch(y_pred, y):
y_pred = torch.argmax(y_pred, axis=1 ).cpu().numpy()
y = y.cpu().numpy()
return accuracy_score(y, y_pred)
 def rand_bbox(size, lam):
W = size[2]
H = size[3]
cut_rat = np.sqrt(1.- lam)
 cut_w = np.int(W * cut_rat)
 cut_h = np.int(H * cut_rat)
 cx = np.random.randint(W)
 cy = np.random.randint(H)
 bbx1 = np.clip(cx - cut_w // 2, 0, W)
 bby1 = np.clip(cy - cut_h // 2, 0, H)
 bbx2 = np.clip(cx + cut_w // 2, 0, W)
 bby2 = np.clip(cy + cut_h // 2, 0, H)
 return bbx1, bby1, bbx2, bby2<train_model>
if run_mode == 'LGBM_KFold':
submission = pd.DataFrame()
submission['SK_ID_CURR'] = test_id
submission['TARGET'] = sub_preds
submission.to_csv('LGBM_SKFold.csv', index=False )
def train(
 params,
model,
optimizer,
criterion,
dataloader,
parent_bar=None, 
):
model.train()
losses = AverageMeter()
metrics = AverageMeter()
for x, y in progress_bar(dataloader, parent=parent_bar):
x = x.to(dtype=torch.float32, device=DEVICE)
 y = y.to(dtype=torch.long, device=DEVICE)
 optimizer.zero_grad()
y_pred = model(x)
 loss = criterion(y_pred, y)
 loss.backward()
optimizer.step()
losses.update(loss.item())
 metrics.update(accuracy_score_torch(y_pred, y))
return losses.avg, metrics.avg
def valid(
 params,
model,
criterion,
dataloader, 
):
model.eval()
losses = AverageMeter()
metrics = AverageMeter()
for x, y in dataloader:
x = x.to(dtype=torch.float32, device=DEVICE)
 y = y.to(dtype=torch.long, device=DEVICE)
 with torch.no_grad() :
y_pred = model(x)
 loss = criterion(y_pred, y)
 losses.update(loss.item())
 metrics.update(accuracy_score_torch(y_pred, y))
return losses.avg, metrics.avg<init_hyperparams>
if run_mode == 'train_estimator_RFR':
estimator = RandomForestRegressor(n_estimators=125,
max_features=0.2,
min_samples_split=2,
min_samples_leaf=75,
n_jobs=-1,
random_state=42,
verbose=0)
 print("
Preparing to train the following estimator:
{}".format(estimator))
start = time()
estimator.fit(X_train, y_train)
 end = time()
print("
Estimator fit time: {} seconds".format(int(round(end - start))))
pred_val = estimator.predict(X_val)
 print("
Estimator prediction score on Validation set: \t{}".format(roc_auc_score(y_val, pred_val)))
 fi = pd.DataFrame()
fi['feature'] = X_train.columns
fi['importance'] = estimator.feature_importances_
display(fi.sort_values(by=['importance'], ascending=False ).head(10))
<split><EOS> 
if 'train_estimator_' in run_mode:
pred_test = estimator.predict(test)
 print("
Preparing prediction for submission.")
 submission = pd.DataFrame()
submission['SK_ID_CURR'] = test_id
submission['TARGET'] = pred_test
submission.head()
file_name = run_mode.split('train_estimator_')[1] + '.csv'
submission.to_csv(file_name, index=False )