Kaggle competition: AILAB ML Training #1
<SOS> metric: categorizationaccuracy Kaggle data source: ailab-ml-training-#1<train_model>
!pip install torchsummary
model_nn.fit(X_train_nn, y_train_nn, epochs=100, batch_size=16 )<compute_train_metric>
%matplotlib inline
y_train_pred_nn = np.squeeze(model_nn.predict(X_train_nn)* std2 + mean2)
 y_test_pred_nn = np.squeeze(model_nn.predict(X_test_nn)* std2 + mean2)
 print(np.any(np.isnan(y_test_pred_nn)== True))
print('RMSLE train : %.6f, test : %.6f' %(np.sqrt(mean_squared_log_error(np.exp(y_train)-1, np.exp(y_train_pred_nn)-1)) , np.sqrt(mean_squared_log_error(np.exp(y_test)-1, np.exp(y_test_pred_nn)-1)))) 
<train_model>
def seed_everything(seed=42):
os.environ['PYTHONHASHSEED'] = str(seed)
 random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def get_logger(
 filename='log',
disable_stream_handler=False,
disable_file_handler=False 
):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 if not disable_stream_handler:
handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 if not disable_file_handler:
handler2 = FileHandler(filename=f"{filename}.log")
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler2)
 return logger
regLasso = Lasso(alpha=0.0007)
 regLasso.fit(X_train, y_train )<compute_test_metric>
logger = get_logger(
 filename='running',
disable_stream_handler=False,
disable_file_handler=False,
)
y_train_pred_lasso = regLasso.predict(X_train)
 y_test_pred_lasso = regLasso.predict(X_test)
 print('RMSLE train : %.6f, test : %.6f' %(np.sqrt(mean_squared_log_error(np.exp(y_train)-1, np.exp(y_train_pred_lasso)-1)) , np.sqrt(mean_squared_log_error(np.exp(y_test)-1, np.exp(y_test_pred_lasso)-1))))<find_best_model_class>
INPUT_DIR = '.. /input/ailab-ml-training-1/'
ARTIFACT_DIR = '.. /input/v1-ailab1-cv/'
PATH = {
'train': os.path.join(INPUT_DIR, 'train.csv'),
'submission': os.path.join(INPUT_DIR, 'sample_submission.csv'),
'train_image_dir': os.path.join(INPUT_DIR, 'train_images/train_images'),
'test_image_dir': os.path.join(INPUT_DIR, 'test_images/test_images'),
'state_dict': os.path.join(ARTIFACT_DIR, 'best_state_dicts.pth'),
'oof': os.path.join(ARTIFACT_DIR, 'oof.npy'),
'predictions': os.path.join(ARTIFACT_DIR, 'predictions.npy'),
}
ID = 'fname'
TARGET = 'label'
SEED = 42
DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
seed_everything(SEED )
def blended_predictions(X, X_nn):
return(2.27101366e-02 * mod_xgb_opt.predict(X)) +(5.76237370e-01 * mod_lgb_opt.predict(X)) \
+(1.90522937e-19 * np.squeeze(model_nn.predict(X_nn)* std2 + mean2)) +(4.02316541e-01 * regLasso.predict(X))
def blended_predictions2(X, X_nn):
return(0.678351173 * mod_lgb_opt.predict(X)) +(0.322875938 * regLasso.predict(X))
def blended_predictions_tune111(X, X_nn, r1, r2, r3):
return(r1 * mod_xgb_opt.predict(X)) +(r2 * mod_lgb_opt.predict(X)) +(r3 * np.squeeze(model_nn.predict(X_nn)* std2 + mean2))<define_variables>
train_df = pd.read_csv(PATH['train'])
 submission_df = pd.read_csv(PATH['submission'] )
<compute_train_metric>
train_df[ID] = train_df[ID].apply(lambda x: os.path.join(PATH['train_image_dir'], x))
y_train_pred_blended = blended_predictions(X_train, X_train_nn)
 y_test_pred_blended = blended_predictions(X_test, X_test_nn)
 print('RMSLE train : %.6f, test : %.6f' %(np.sqrt(mean_squared_log_error(np.exp(y_train)-1, np.exp(y_train_pred_blended)-1)) , np.sqrt(mean_squared_log_error(np.exp(y_test)-1, np.exp(y_test_pred_blended)-1)))) 
<feature_engineering>
def softmax(logits):
return np.exp(logits)/ np.sum(np.exp(logits), axis=1, keepdims=True )
XX_TEST_nn = copy.deepcopy(XX_TEST)
 XX_TEST_nn =(XX_TEST_nn - mean1)/ std1<predict_on_test>
predictions = np.load(PATH['predictions'])
 predictions = softmax(predictions)
 predictions_label, predictions_proba = np.argmax(predictions, axis=1), np.max(predictions, axis=1)
 pseudo_df = submission_df.copy()
pseudo_df[TARGET] = predictions_label
pseudo_df['proba'] = predictions_proba
pseudo_df = pseudo_df.loc[pseudo_df['proba'] > 0.999, :]
pseudo_df = pseudo_df.drop('proba', axis=1)
 pseudo_df[ID] = pseudo_df[ID].apply(lambda x: os.path.join(PATH['test_image_dir'], x))
TEST_pred = blended_predictions(XX_TEST, XX_TEST_nn)
 print(TEST_pred )<prepare_output>
TEST_pred_raw = np.exp(TEST_pred)-1
print(TEST_pred_raw )<prepare_output>
class KmnistDataset(Dataset):
def __init__(
 self,
paths,
labels,
transform=None,
with_memory_cache=False, 
):
super().__init__()
self.paths = paths
self.labels = labels
self.transform = transform
self.with_memory_cache = with_memory_cache
if with_memory_cache:
self.images = [None,] * len(paths)
 def load_image(self, path):
image = cv2.imread(path)
 return image
def __len__(self):
return len(self.paths)
 def __getitem__(self, idx):
if self.with_memory_cache:
image = self.images[idx]
if image is None:
path = self.paths[idx]
image = self.load_image(path)
 self.images[idx] = image
else:
path = self.paths[idx]
image = self.load_image(path)
 label = self.labels[idx]
if self.transform is not None:
image = self.transform(image=image)['image']
return image, label
TEST_pred_raw2 = np.exp(TEST_pred_2)-1
print(TEST_pred_raw2 )<create_dataframe>
def get_dataloader(
 X,
Y,
transform=None,
with_memory_cache=False,
batch_size=32,
shuffle=False,
num_workers=0,
pin_memory=True, 
):
dataset = KmnistDataset(
 X,
Y,
transform=transform,
with_memory_cache=with_memory_cache, 
)
 loader = DataLoader(
 dataset=dataset,
batch_size=batch_size,
shuffle=shuffle,
num_workers=num_workers,
pin_memory=pin_memory, 
)
 return loader
submission= pd.DataFrame()
submission['id'] = test['Id']
submission['SalePrice'] = TEST_pred_raw
submission2= pd.DataFrame()
submission2['id'] = test['Id']
submission2['SalePrice'] = TEST_pred_raw2<save_to_csv>
num_samples = 20
sample = train_df.groupby(TARGET ).apply(lambda df: df.sample(num_samples))
fnames = sample[ID].to_list()
labels = sample[TARGET].to_list()
submission.to_csv('submission.csv', index=False )<save_to_csv>
def conv3x3(in_planes, out_planes, stride=1, padding=1):
return nn.Conv2d(in_planes, out_planes, kernel_size=3,
stride=stride, padding=padding, bias=False)
 def conv1x1(in_planes, out_planes, stride=1):
return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
 class SSELayer(nn.Module):
def __init__(self, channel):
super().__init__()
self.conv = nn.Sequential(
 conv1x1(channel, 1),
nn.Sigmoid() 
)
 def forward(self, x):
y = self.conv(x)
 return x * y.expand_as(x)
 class CSELayer(nn.Module):
def __init__(self, channel, reduction=16):
super().__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc = nn.Sequential(
 nn.Linear(channel, channel // reduction, bias=False),
nn.ReLU(inplace=True),
nn.Linear(channel // reduction, channel, bias=False),
nn.Sigmoid() 
)
 def forward(self, x):
b, c, _, _ = x.size()
y = self.avg_pool(x ).view(b, c)
 y = self.fc(y ).view(b, c, 1, 1)
 return x * y.expand_as(x)
 class SCSELayer(nn.Module):
def __init__(self, channel, reduction=16):
super().__init__()
self.sse = SSELayer(channel)
 self.cse = CSELayer(channel, reduction)
 def forward(self, x):
return self.sse(x)+ self.cse(x)
 def se_layer(channel, se_type=None, reduction=16):
if se_type is None:
return nn.Identity()
elif se_type == 'cse':
return CSELayer(channel, reduction)
 elif se_type == 'sse':
return SSELayer(channel)
 elif se_type == 'scse':
return SCSELayer(channel, reduction)
 else:
raise NotImplementedError
class BasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None,
norm_layer=None, se_type=None, reduction=16):
super().__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
self.conv1 = conv3x3(inplanes, planes, stride)
 self.bn1 = norm_layer(planes)
 self.relu = nn.ReLU(inplace=True)
 self.conv2 = conv3x3(planes, planes)
 self.bn2 = norm_layer(planes)
 self.se = se_layer(planes, se_type=se_type, reduction=reduction)
 self.downsample = downsample
def forward(self, x):
identity = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.se(out)
 if self.downsample is not None:
identity = self.downsample(x)
 out += identity
out = self.relu(out)
 return out
class ResNet(nn.Module):
def __init__(self, block, layers, num_classes=10, width_per_group=64,
se_type=None, reduction=16):
super(ResNet, self ).__init__()
norm_layer = nn.BatchNorm2d
self._norm_layer = norm_layer
self.inplanes = 64
self.base_width = width_per_group
self.se_type = se_type
self.reduction = reduction
self.conv1 = nn.Conv2d(1, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False)
 self.bn1 = norm_layer(self.inplanes)
 self.relu = nn.ReLU(inplace=True)
 self.layer1 = self._make_layer(block, 64, layers[0])
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
 self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
 self.avgpool = nn.AdaptiveAvgPool2d(( 1, 1))
self.fc = nn.Linear(512 * block.expansion, num_classes)
 for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
 elif isinstance(m,(nn.BatchNorm2d, nn.GroupNorm)) :
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 def _make_layer(self, block, planes, blocks, stride=1):
norm_layer = self._norm_layer
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 conv1x1(self.inplanes, planes * block.expansion, stride),
norm_layer(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, stride, downsample=downsample, norm_layer=norm_layer,
se_type=self.se_type, reduction=self.reduction))
self.inplanes = planes * block.expansion
for _ in range(1, blocks):
layers.append(block(self.inplanes, planes, norm_layer=norm_layer,
se_type=self.se_type, reduction=self.reduction))
return nn.Sequential(*layers)
 def _forward_impl(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = self.relu(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.avgpool(x)
 x = torch.flatten(x, 1)
 x = self.fc(x)
 return x
def forward(self, x):
return self._forward_impl(x)
 def small_resnet18(se_type='scse'):
return ResNet(BasicBlock, [2, 2, 2, 2], se_type=se_type)
 def small_resnet34(se_type='scse'):
return ResNet(BasicBlock, [3, 4, 6, 3], se_type=se_type )
submission.to_csv('submission.csv', index=False )<save_to_csv>
class AverageMeter:
def __init__(self):
self.reset()
def reset(self):
self.val = 0.0
self.avg = 0.0
self.sum = 0.0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def accuracy_score_torch(y_pred, y):
y_pred = torch.argmax(y_pred, axis=1 ).cpu().numpy()
y = y.cpu().numpy()
return accuracy_score(y, y_pred)
 def rand_bbox(size, lam):
W = size[2]
H = size[3]
cut_rat = np.sqrt(1.- lam)
 cut_w = np.int(W * cut_rat)
 cut_h = np.int(H * cut_rat)
 cx = np.random.randint(W)
 cy = np.random.randint(H)
 bbx1 = np.clip(cx - cut_w // 2, 0, W)
 bby1 = np.clip(cy - cut_h // 2, 0, H)
 bbx2 = np.clip(cx + cut_w // 2, 0, W)
 bby2 = np.clip(cy + cut_h // 2, 0, H)
 return bbx1, bby1, bbx2, bby2
submission2.to_csv('submission2.csv', index=False )<set_options>
def train(
 params,
model,
optimizer,
criterion,
dataloader,
parent_bar=None, 
):
model.train()
losses = AverageMeter()
metrics = AverageMeter()
for x, y in progress_bar(dataloader, parent=parent_bar):
x = x.to(dtype=torch.float32, device=DEVICE)
 y = y.to(dtype=torch.long, device=DEVICE)
 optimizer.zero_grad()
y_pred = model(x)
 loss = criterion(y_pred, y)
 loss.backward()
optimizer.step()
losses.update(loss.item())
 metrics.update(accuracy_score_torch(y_pred, y))
return losses.avg, metrics.avg
def valid(
 params,
model,
criterion,
dataloader, 
):
model.eval()
losses = AverageMeter()
metrics = AverageMeter()
for x, y in dataloader:
x = x.to(dtype=torch.float32, device=DEVICE)
 y = y.to(dtype=torch.long, device=DEVICE)
 with torch.no_grad() :
y_pred = model(x)
 loss = criterion(y_pred, y)
 losses.update(loss.item())
 metrics.update(accuracy_score_torch(y_pred, y))
return losses.avg, metrics.avg
warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', 10000)
 pd.set_option('display.max_rows', 10000 )<load_from_csv>
params = {
'n_splits': 5,
'batch_size': 64,
'test_batch_size': 128,
'lr': 1e-3,
'weight_decay': 1e-5,
'epochs': 25,
'restart': 25,
}
df_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 df_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )<prepare_x_and_y>
kf = StratifiedKFold(n_splits=params['n_splits'], random_state=SEED, shuffle=True)
 oof = np.zeros(( len(train_df), 10))
best_state_dicts = []
logger.info('** start kfold training **')
 for i,(dev_idx, val_idx)in enumerate(kf.split(train_df[ID], train_df[TARGET])) :
logger.info(f'[fold: {i}]')
 dev_df = train_df.iloc[dev_idx, :].reset_index(drop=True)
 val_df = train_df.iloc[val_idx, :].reset_index(drop=True)
 dev_df = pd.concat([dev_df, pseudo_df], axis=0 ).reset_index(drop=True)
 dev_transform = A.Compose([
A.Rotate(limit=5, p=1.0),
A.RandomResizedCrop(IMSIZE, IMSIZE, scale=(0.9, 1.0), ratio=(0.9, 1.1), p=1.0),
A.Cutout(num_holes=1, max_h_size=IMSIZE//2, max_w_size=IMSIZE//2, fill_value=48.89935, p=1.0),
A.Normalize(( 0.485, 0.456, 0.406),(0.229, 0.224, 0.225)) ,
ToTensorV2() ,
])
 val_transform = A.Compose([
A.Resize(IMSIZE, IMSIZE, p=1),
A.Normalize(( 0.485, 0.456, 0.406),(0.229, 0.224, 0.225)) ,
ToTensorV2() ,
])
 dev_dataloader = get_dataloader(
 dev_df[ID],
dev_df[TARGET],
transform=dev_transform,
with_memory_cache=True,
batch_size=params['batch_size'],
shuffle=True,
pin_memory=True, 
)
 val_dataloader = get_dataloader(
 val_df[ID],
val_df[TARGET],
transform=val_transform,
with_memory_cache=True,
batch_size=params['test_batch_size'],
shuffle=False,
pin_memory=True, 
)
 model = Model().to(DEVICE)
 optimizer = Adam(model.parameters() , lr=params['lr'], weight_decay=params['weight_decay'])
 scheduler = CosineAnnealingWarmRestarts(optimizer, params['restart'])
 criterion = nn.CrossEntropyLoss()
best_loss = np.inf
best_state_dict = model.state_dict()
mb = master_bar(range(params['epochs']))
for epoch in mb:
dev_loss, dev_metric = train(
 params, model, optimizer, criterion, dev_dataloader, parent_bar=mb)
 val_loss, val_metric = valid(
 params, model, criterion, val_dataloader)
 if val_loss < best_loss:
best_loss = val_loss
best_state_dict = model.state_dict()
msg =(
 'epoch: {}/{}'
' - lr: {:.6f}'
' - loss: {:.5f}'
' - acc: {:.5f}'
' - val_loss: {:.5f}'
' - val_acc: {:.5f}'
).format(
 epoch + 1,
params['epochs'],
optimizer.state_dict() ['param_groups'][0]['lr'],
dev_loss,
dev_metric,
val_loss,
val_metric, 
)
 logger.info(msg)
 if scheduler is not None:
scheduler.step()
best_state_dicts.append(best_state_dict)
 model.load_state_dict(best_state_dict)
 model.eval()
_oof = []
for x, _ in val_dataloader:
x = x.to(dtype=torch.float32, device=DEVICE)
 with torch.no_grad() :
y_pred = model(x)
 _oof.append(y_pred.cpu().numpy())
 oof[val_idx] = np.concatenate(_oof )
y = df_train[['Id','SalePrice']]
df_train = df_train.drop('SalePrice',axis=1 )<concatenate>
cv_score = accuracy_score(train_df[TARGET], np.argmax(oof, axis=1))
logger.info(f'CV: {cv_score:.5f}' )
concat_df = [df_train,df_test]
df_all = pd.concat(concat_df ).reset_index(drop=True )<categorify>
test_transform = A.Compose([
A.Resize(IMSIZE, IMSIZE, p=1),
A.Normalize(( 0.485, 0.456, 0.406),(0.229, 0.224, 0.225)) ,
ToTensorV2() ,
])
 test_dataloader = get_dataloader(
 submission_df[ID].apply(lambda x: os.path.join(PATH['test_image_dir'], x)) ,
submission_df[TARGET],
transform=test_transform,
with_memory_cache=True,
batch_size=params['test_batch_size'],
shuffle=False,
pin_memory=True,
)
df_dummy = pd.get_dummies(df_all['MSZoning'],prefix='MSZoning',dummy_na=True)
 df_all = pd.concat([df_all,df_dummy],axis=1)
 del df_all['MSZoning']<data_type_conversions>
predictions = np.zeros(( len(submission_df), 10))
for state_dict in best_state_dicts:
model = Model().to(DEVICE)
 model.load_state_dict(state_dict)
 model.eval()
_predictions = []
for x, _ in test_dataloader:
x = x.to(dtype=torch.float32, device=DEVICE)
 with torch.no_grad() :
y_pred = model(x)
 _predictions.append(y_pred.cpu().numpy())
 _predictions = np.concatenate(_predictions)
 predictions += _predictions / len(best_state_dicts )
df_all['LotFrontage'] = df_all['LotFrontage'].fillna(df_all['LotFrontage'].mean() )<categorify>
np.save('oof', oof)
 np.save('predictions', predictions)
 saved_best_state_dicts = {}
for i, bsd in enumerate(best_state_dicts):
saved_best_state_dicts[f'f{i}'] = bsd
torch.save(saved_best_state_dicts, 'best_state_dicts.pth')
 submission_df[TARGET] = np.argmax(predictions, axis=1 ).tolist()
<feature_engineering><EOS> 
submission_df.to_csv('submission.csv', index=False)
 FileLink('submission.csv' )