Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<save_to_csv>
%matplotlib inline
sub_df = pd.DataFrame(test_ID)
 sub_df["PredictedLogRevenue"] = sub_preds
sub_df.to_csv("stacked_result.csv", index=False )<set_options>
seed = 520
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.benchmark = True
use_gpu = True
InteractiveShell.ast_node_interactivity='all'<load_from_csv>
labels = pd.read_csv('.. /input/train_labels.csv')
 sub = pd.read_csv('.. /input/sample_submission.csv')
 train_path = '.. /input/train/'
test_path = '.. /input/test/'
def load_dataset(path,nrows=None):
cols=['channelGrouping', 'date', 'device', 'fullVisitorId', 'geoNetwork',
'socialEngagementType', 'totals', 'trafficSource', 'visitId','customDimensions',
'visitNumber', 'visitStartTime']
json_cols=['device','geoNetwork', 'totals', 'trafficSource']
df = pd.read_csv(path,
converters={column: json.loads for column in json_cols},
dtype={'fullVisitorId': 'str'},
nrows=nrows, usecols=cols)
 for column in json_cols:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}.{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 return df<load_from_csv>
labels.label.value_counts()
%%time
train=load_dataset('.. /input/train_v2.csv' )<load_from_csv>
train, val = train_test_split(labels, stratify=labels.label, test_size=0.1)
 len(val),len(train )
%%time
test=load_dataset('.. /input/test_v2.csv' )<count_unique_values>
class MyDataset(Dataset):
def __init__(self, df_data, data_dir = './', transform=None):
super().__init__()
self.df = df_data.values
self.data_dir = data_dir
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
img_name,label = self.df[index]
img_path = os.path.join(self.data_dir, img_name+'.tif')
 image = mask(cv2.imread(img_path))
if self.transform is not None:
image = self.transform(image)
 return image, label
for cols in train.columns:
if(train[cols].nunique() == 1):
train[cols].value_counts()
del train[cols]
for cols in test.columns:
if(test[cols].nunique() == 1):
del test[cols]
train['set']='Train'
test['set']='Test'
data= pd.concat([train,test],axis=0 )<drop_column>
batch_size = 128
trans_train = transforms.Compose([transforms.ToPILImage() ,
transforms.Pad(64, padding_mode='reflect'),
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
transforms.RandomRotation(20),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])
 trans_valid = transforms.Compose([transforms.ToPILImage() ,
transforms.Pad(64, padding_mode='reflect'),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])
 dataset_train = MyDataset(df_data=train, data_dir=train_path, transform=trans_train)
 dataset_valid = MyDataset(df_data=val, data_dir=train_path, transform=trans_valid)
 loader_train = DataLoader(dataset = dataset_train, batch_size=batch_size, shuffle=True, num_workers=0)
 loader_valid = DataLoader(dataset = dataset_valid, batch_size=batch_size//2, shuffle=False, num_workers=0 )
data.drop('customDimensions',inplace=True,axis=1)
 data.head(5 )<count_missing_values>
model = models.resnet34(pretrained=True)
for col in data.columns:
if data[col].isnull().sum() > 0:
rate = data[col].isnull().sum() * 100 / data.shape[0]
print(f'Column {col} has {rate:.4f}% missing values.')
 if data[col].dtype == 'object':
if(data[col] == 'not available in demo dataset' ).sum() > 0:
rate =(data[col] == 'not available in demo dataset' ).sum() * 100 / data.shape[0]
print(f'Column {col} has {rate:.4f}% values not available in dataset.' )<feature_engineering>
for para in list(model.parameters()):
para.requires_grad=False
for para in list(model.layer3.parameters()):
para.requires_grad=True
for para in list(model.layer4.parameters()):
para.requires_grad=True
data['date']=data['date'].apply(lambda x: pd.to_datetime(str(x),format='%Y%m%d'))
data['year']=data['date'].dt.year
data['month']=data['date'].dt.month
data['day']=data['date'].dt.day
data['weekday']=data['date'].dt.weekday<feature_engineering>
num_ftrs = model.fc.in_features
model.fc = nn.Sequential(
 nn.Dropout(0.5),
nn.Linear(num_ftrs, 2)
 )
not_null=pd.notnull(data['totals.transactionRevenue'] ).sum()
print(f'No of records generating the transactions : {not_null}')
 null_val=round(pd.isnull(data['totals.transactionRevenue'] ).sum() /data.shape[0],4)
 print(f'No of records with Null transaction : {null_val}')
 data['totals.transactionRevenue']=data['totals.transactionRevenue'].fillna('0' )<drop_column>
params_to_update = []
for name,param in model.named_parameters() :
if param.requires_grad == True:
params_to_update.append(param)
 print("\t",name )
data.columns
del data['visitId']
del data['visitNumber']<data_type_conversions>
num_epochs = 6
early_stopping = 4
data['totals.hits']=data['totals.hits'].fillna(1)
 data['totals.hits']=data['totals.hits'].astype('Int64' )<data_type_conversions>
criterion = nn.BCEWithLogitsLoss()
optimizer = optim.Adam(params_to_update, lr=0.001)
 scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.25)
 best_val_acc = 0
best_epoch = 0
epoch_since_best = 0
for epoch in range(num_epochs):
at = time.time()
scheduler.step()
model.train()
train_acc = []
train_loss = []
for i, data in enumerate(loader_train):
if i%10 == 0:
print('.',end='')
 inputs, labels = data
if use_gpu:
inputs, labels = inputs.cuda() , labels.cuda()
optimizer.zero_grad()
outputs = model(inputs)
 loss = criterion(outputs[:,1], labels.float())
 loss.backward()
optimizer.step()
a = labels.data.cpu().numpy()
b = outputs[:,-1].detach().cpu().numpy()
train_acc.append(roc_auc_score(a, b))
train_loss.append(loss.item())
 model.eval()
valid_acc = []
val_loss = []
for _, data in enumerate(loader_valid):
inputs, labels = data
if use_gpu:
inputs, labels = inputs.cuda() , labels.cuda()
optimizer.zero_grad()
outputs = model(inputs)
 loss = criterion(outputs[:,1], labels.float())
 a = labels.data.cpu().numpy()
b = outputs[:,-1].detach().cpu().numpy()
valid_acc.append(roc_auc_score(a, b))
val_loss.append(loss.item())
 print()
bt = time.time()
print('[Epoch %d] train loss %.6f train acc %.6f valid loss %.6f valid acc %.6f time %.6f' %(
 epoch, np.mean(train_loss), np.mean(train_acc), np.mean(val_loss), np.mean(valid_acc),bt-at))
valid_acc = np.mean(valid_acc)
 if valid_acc > best_val_acc:
best_val_acc = valid_acc
best_epoch = epoch
epoch_since_best = 0
print('save model...')
 torch.save(model.state_dict() , 'model.pth')
 print('saved.')
 else:
epoch_since_best += 1
if epoch_since_best > early_stopping:
break
print('Finished Training')
 print('best_epoch: %d, best_val_acc %.6f' %(best_epoch, best_val_acc))
data['totals.pageviews']=data['totals.pageviews'].fillna(1)
 data['totals.pageviews']=data['totals.pageviews'].astype('Int64')
 data['totals.pageviews'].dtypes 
<data_type_conversions>
trans_train = transforms.Compose([trainsforms.ToPILImage() ,
transforms.Pad(64, padding_mode='reflect'),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])
 trans_valid = transforms.Compose([trainsforms.ToPILImage() ,
transforms.Pad(64, padding_mode='reflect'),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])
 dataset_train = MyDataset(df_data=train, data_dir=train_path, transform=trans_train)
 dataset_valid = MyDataset(df_data=val, data_dir=train_path, transform=trans_valid)
 loader_train = DataLoader(dataset = dataset_train, batch_size=batch_size, shuffle=True, num_workers=0)
 loader_valid = DataLoader(dataset = dataset_valid, batch_size=batch_size//2, shuffle=False, num_workers=0)
 optimizer = optim.Adam(params_to_update, lr=0.0001)
 scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.25)
 for epoch in range(6,10):
at = time.time()
scheduler.step()
model.train()
train_acc = []
train_loss = []
for i, data in enumerate(loader_train):
if i%10 == 0:
print('.',end='')
 inputs, labels = data
if use_gpu:
inputs, labels = inputs.cuda() , labels.cuda()
optimizer.zero_grad()
outputs = model(inputs)
 loss = criterion(outputs[:,1], labels.float())
 loss.backward()
optimizer.step()
a = labels.data.cpu().numpy()
b = outputs[:,-1].detach().cpu().numpy()
train_acc.append(roc_auc_score(a, b))
train_loss.append(loss.item())
 model.eval()
valid_acc = []
val_loss = []
for _, data in enumerate(loader_valid):
inputs, labels = data
if use_gpu:
inputs, labels = inputs.cuda() , labels.cuda()
optimizer.zero_grad()
outputs = model(inputs)
 loss = criterion(outputs[:,1], labels.float())
 a = labels.data.cpu().numpy()
b = outputs[:,-1].detach().cpu().numpy()
valid_acc.append(roc_auc_score(a, b))
val_loss.append(loss.item())
 print()
bt = time.time()
print('[Epoch %d] train loss %.6f train acc %.6f valid loss %.6f valid acc %.6f time %.6f' %(
 epoch, np.mean(train_loss), np.mean(train_acc), np.mean(val_loss), np.mean(valid_acc),bt-at))
valid_acc = np.mean(valid_acc)
 if valid_acc > best_val_acc:
best_val_acc = valid_acc
best_epoch = epoch
epoch_since_best = 0
print('save model...')
 torch.save(model.state_dict() , 'model.pth')
 print('saved.')
 else:
epoch_since_best += 1
if epoch_since_best > early_stopping:
break
print('Finished Training')
 print('best_epoch: %d, best_val_acc %.6f' %(best_epoch, best_val_acc))
<groupby><EOS> 
dataset_valid = MyDataset(df_data=sub, data_dir=test_path, transform=trans_valid)
 loader_test = DataLoader(dataset = dataset_valid, batch_size=32, shuffle=False, num_workers=0)
 model.eval()
preds = []
for batch_i,(data, target)in tqdm(enumerate(loader_test)) :
data, target = data.cuda() , target.cuda()
output = model(data)
 pr = output[:,1].detach().cpu().numpy()
for i in pr:
preds.append(i)
 sub['label'] = preds
sub.to_csv('sub.csv', index=False )