Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<load_from_csv>
!pip install albumentations
!pip install pretrainedmodels
!pip install torchsummary
%matplotlib inline
train_on_gpu = True
batch_size = 64
batch_size_test = 64
num_workers = 4
target_names = ['class 0', 'class 1']
def read_csv(files):
data_train = pd.read_csv(files[0],
dtype={
'StateHoliday': 'category',
'SchoolHoliday': 'int'},
parse_dates=['Date'])
 data_test = pd.read_csv(files[1],
dtype={
'StateHoliday': 'category',
'SchoolHoliday': 'int'},
parse_dates=['Date'])
 data_store = pd.read_csv(files[2],
dtype={
'StoreType': 'category',
'Assortment': 'category',
'CompetitionOpenSinceMonth': float,
'CompetitionOpenSinceYear': float,
'Promo2': float,
'Promo2SinceWeek': float,
'Promo2SinceYear': float,
'PromoInterval': str})
 return data_train, data_test, data_store
def combine(data, data_store):
return pd.merge(data, data_store, on='Store', how='left')
 def preprocess(data_train, data_test):
data_train_ = data_train.copy()
data_test_ = data_test.copy()
data_train_ = data_train_[(data_train_['Sales'] > 0)&(data_train_['Open'] != 0)]
data_test_['Open'] = data_test_['Open'].fillna(1)
 def process(data):
data['Year'] = data['Date'].dt.year
data['Month'] = data['Date'].dt.month
data['Day'] = data['Date'].dt.day
data['DayOfWeek'] = data['Date'].dt.dayofweek
data['DayOfYear'] = data['Date'].dt.dayofyear
data['WeekOfYear'] = data['Date'].dt.weekofyear
data['Quarter'] =(data['Date'].dt.month - 1)// 3 + 1
mappings = {'0': 0, 'a': 1, 'b': 2, 'c': 3, 'd': 4}
data['StoreType'].replace(mappings, inplace=True)
 data['Assortment'].replace(mappings, inplace=True)
 data['StateHoliday'].replace(mappings, inplace=True)
 data['CompetitionDistance'] = data['CompetitionDistance'].fillna(data['CompetitionDistance'].median())
 data['CompetitionOpen'] = 12 *(data['Year'] - data['CompetitionOpenSinceYear'])+(data['Month'] - data['CompetitionOpenSinceMonth'] ).apply(lambda x: x if x > 0 else 0)
 data['Promo2Open'] = 12 *(data['Year'] - data['Promo2SinceYear'])+(data['WeekOfYear'] - data['Promo2SinceWeek'])/ 4.0
data['Promo2Open'] = data['Promo2Open'].apply(lambda x: x if x > 0 else 0)
 data['PromoInterval'] = data['PromoInterval'].fillna('')
 data['InPromoMonth'] = data.apply(lambda x: 1 if(x['Date'].strftime('%b')if not x['Date'].strftime('%b')== 'Sep' else 'Sept')in x['PromoInterval'].split(',')else 0, axis=1)
 data_meanlog_salesbystore = data_train_.groupby(['Store'])['Sales'].mean().reset_index(name='MeanLogSalesByStore')
 data_meanlog_salesbystore['MeanLogSalesByStore'] = np.log1p(data_meanlog_salesbystore['MeanLogSalesByStore'])
 data = data.merge(data_meanlog_salesbystore, on=['Store'], how='left', validate='m:1')
 data_meanlog_salesbydow = data_train_.groupby(['DayOfWeek'])['Sales'].mean().reset_index(name='MeanLogSalesByDOW')
 data_meanlog_salesbydow['MeanLogSalesByDOW'] = np.log1p(data_meanlog_salesbystore['MeanLogSalesByStore'])
 data = data.merge(data_meanlog_salesbydow, on=['DayOfWeek'], how='left', validate='m:1')
 data_meanlog_salesbymonth = data_train_.groupby(['Month'])['Sales'].mean().reset_index(name='MeanLogSalesByMonth')
 data_meanlog_salesbymonth['MeanLogSalesByMonth'] = np.log1p(data_meanlog_salesbymonth['MeanLogSalesByMonth'])
 data = data.merge(data_meanlog_salesbymonth, on=['Month'], how='left', validate='m:1')
 return data
features = [
'Store', 'DayOfWeek', 'Year', 'Month', 'Day', 'DayOfYear', 'WeekOfYear', 'Quarter', 'Open', 'Promo',
'StateHoliday', 'SchoolHoliday', 'StoreType', 'Assortment',
'CompetitionDistance', 'CompetitionOpenSinceMonth', 'CompetitionOpenSinceYear', 'CompetitionOpen',
'Promo2', 'Promo2SinceWeek', 'Promo2SinceYear', 'Promo2Open', 'InPromoMonth',
'MeanLogSalesByStore', 'MeanLogSalesByDOW', 'MeanLogSalesByMonth']
data_train_ = process(data_train_)
 data_test_ = process(data_test_)
 return(data_train_[features], np.log1p(data_train_['Sales'])) , data_test_[features]<load_from_csv>
class MyDataset(Dataset):
def __init__(self, datatype='train', df=None,transform=None ,augument_=True,dataloc=".. /input/histopathologic-cancer-detection/train/"):
self.datatype = datatype
self.df = df
self.augument = augument_
self.transform = transform
self.dataloc = dataloc
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
label = self.df[idx][1]
img_name = self.df[idx][0]+'.tif'
img_dir = os.path.join(self.dataloc,img_name)
 img = cv2.imread(img_dir)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = self.transform(image=img)
 return img['image'],label
pd.set_option('display.max_columns', None)
 pd.set_option('display.expand_frame_repr', False)
 pd.set_option('max_colwidth', -1)
 data_train, data_test, data_store = read_csv(['.. /input/train.csv', '.. /input/test.csv', '.. /input/store.csv'])
 data_train = combine(data_train, data_store)
 data_test = combine(data_test, data_store)
(X_train, y_train), X_test = preprocess(data_train, data_test )<compute_test_metric>
data_transforms = albumentations.Compose([
albumentations.Resize(96, 96),
albumentations.RandomRotate90(p=0.5),
albumentations.Transpose(p=0.5),
albumentations.Flip(p=0.5),
albumentations.OneOf([
albumentations.CLAHE(clip_limit=2), albumentations.IAASharpen() , albumentations.IAAEmboss() ,
albumentations.RandomBrightness() , albumentations.RandomContrast() ,
albumentations.JpegCompression() , albumentations.Blur() , albumentations.GaussNoise() ], p=0.5),
albumentations.HueSaturationValue(p=0.5),
albumentations.ShiftScaleRotate(shift_limit=0.15, scale_limit=0.15, rotate_limit=45, p=0.5),
albumentations.Normalize() ,
AT.ToTensor()
])
 data_transforms_test = albumentations.Compose([
albumentations.Resize(96, 96),
albumentations.Normalize() ,
AT.ToTensor()
] )
def rmspe(y_true, y_pred):
err = np.sqrt(np.mean(( 1 - y_pred / y_true)** 2))
return err
def rmspe_xgb(y_pred, y_true):
y_true = y_true.get_label()
err = rmspe(y_true, y_pred)
 return 'rmspe', err<train_model>
class Densenet169(nn.Module):
def __init__(self, pretrained=True):
super(Densenet169, self ).__init__()
self.model = models.densenet169(num_classes=1000,pretrained=pretrained)
 self.linear = nn.Linear(1000+2, 16)
 self.bn = nn.BatchNorm1d(16)
 self.dropout = nn.Dropout(0.2)
 self.elu = nn.ELU()
self.out = nn.Linear(16, 1)
 self.sig = nn.Sigmoid()
def forward(self, x):
out = self.model(x)
 batch = out.shape[0]
max_pool, _ = torch.max(out, 1, keepdim=True)
 avg_pool = torch.mean(out, 1, keepdim=True)
 out = out.view(batch, -1)
 conc = torch.cat(( out, max_pool, avg_pool), 1)
 conc = self.linear(conc)
 conc = self.elu(conc)
 conc = self.bn(conc)
 conc = self.dropout(conc)
 res = self.out(conc)
 res = self.sig(res)
 return res
model_conv = Densenet169(pretrained=False)
 model_conv.cuda()
class Model:
def __init__(self, params=None, **kwargs):
self.params = params
self.kwargs = kwargs
def train(self, X, y):
X_train, X_test, y_train, y_test = X[41088:], X[:41088], y[41088:], y[:41088]
dtrain = xgb.DMatrix(X_train, label=y_train)
 dvalid = xgb.DMatrix(X_test, label=y_test)
 watchlist = [(dvalid, 'eval'),(dtrain, 'train')]
self.bst = xgb.train(self.params, dtrain, self.kwargs['num_boost_round'], evals=watchlist,
feval=rmspe_xgb, early_stopping_rounds=self.kwargs['early_stopping_rounds'],
verbose_eval=True)
 def predict(self, X, weight=0.995):
y_pred = np.expm1(weight * self.bst.predict(xgb.DMatrix(X)))
 return y_pred
def save_model(self, filename):
joblib.dump(self.bst, filename)
 def load_model(self, filename):
self.bst = joblib.load(filename )<train_model>
model_conv.load_state_dict(torch.load(".. /input/des-model1/model(4 ).pt"))
params = {
'eta': 0.03,
'max_depth': 10,
'subsample': 0.8,
'colsample_bytree': 0.7,
'lambda': 0.2,
'silent': 1,
'seed': 12
}
m = Model(params, num_boost_round=5000, early_stopping_rounds=50)
 m.train(X_train, y_train )<save_to_csv>
test = pd.read_csv('.. /input/histopathologic-cancer-detection/sample_submission.csv')
 test_dataset = MyDataset(datatype='test',df=test.values,transform=data_transforms_test,augument_=False,dataloc=".. /input/histopathologic-cancer-detection/test/")
 test_loader = DataLoader(test_dataset, batch_size=128, num_workers=num_workers, pin_memory=True,shuffle=False )
weight = 1.0
y_pred = m.predict(X_test, weight=weight)
 result = pd.DataFrame({'Id': data_test['Id'], 'Sales': y_pred})
 result.to_csv('submission.csv', index=False)
 result<load_from_csv>
Sig = nn.Sigmoid()
model_conv.eval()
preds = []
with torch.no_grad() :
for batch_i,(data, target)in tqdm(enumerate(test_loader),total=len(test_loader)) :
data, target = data.cuda() , target.cuda()
output = model_conv(data)
 pr = output.detach().cpu().numpy()
for i in pr:
preds.append(i[0] )
store = pd.read_csv(".. /input/store.csv")
 train = pd.read_csv(".. /input/train.csv",parse_dates=[2])
 test = pd.read_csv(".. /input/test.csv",parse_dates=[3] )<count_missing_values>
a = np.array(preds)
store.isnull().sum()<count_values>
sub = pd.read_csv('.. /input/histopathologic-cancer-detection/sample_submission.csv')
 test_preds = pd.DataFrame({'imgs': test["id"].values.tolist() , 'preds': a})
 test_preds['imgs'] = test_preds['imgs'].apply(lambda x: x.split('.')[0])
 sub = pd.merge(sub, test_preds, left_on='id', right_on='imgs')
 sub
store.PromoInterval.value_counts()<correct_missing_values>
sub = sub[['id', 'preds']]
sub.columns = ['id', 'label']
sub.to_csv('sub.csv', index=False)