Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<feature_engineering>
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 print(device)
 warnings.filterwarnings('ignore')
 PATH=".. /input/"
N_SPLITS = 10
SEED_SKF = 4221
def str_clean(x):
return re.sub(re.compile('\W'),' ',x.lower() ).strip()
shops['shop_name'] = shops['shop_name'].apply(str_clean)
 itemcat['item_category_name'] = itemcat['item_category_name'].apply(str_clean )<feature_engineering>
def merge_train_test(df_train, df_test):
if "target" not in df_test.columns.values:
df_test["target"] = -1
res = pd.concat([df_train, df_test])
 res.reset_index(inplace=True, drop=True)
 return res
def split_train_test(df):
df_train = df[df["target"] >= 0]
df_test = df[df["target"] <= -1]
df_train.reset_index(inplace=True, drop=True)
 df_test.reset_index(inplace=True, drop=True)
 assert list(df_train["ID_code"].values)== [f"train_{i}" for i in range(200000)]
assert list(df_test["ID_code"].values)== [f"test_{i}" for i in range(200000)]
return df_train, df_test
items['item_name'] = items['item_name'].apply(str_clean )<feature_engineering>
%%time
train_df = pd.read_csv(PATH+"train.csv")
 test_df = pd.read_csv(PATH+"test.csv" )
duplicate_shops = {57:0,58:1,40:39,11:10}
def fix_shop_id(x):
if x not in duplicate_shops.keys() :
return x
return duplicate_shops[x]
train['shop_id'] = train['shop_id'].apply(fix_shop_id)
 test['shop_id'] = test['shop_id'].apply(fix_shop_id)
 shops.drop(shops[shops.shop_id.isin(duplicate_shops.keys())].index,inplace = True )<feature_engineering>
class CountEncoder:
def fit(self, series):
self.counts = series.groupby(series ).count()
def transform(self, series):
return series.map(self.counts ).fillna(0 ).astype(np.int16 )
shops['city'] = shops.shop_name.apply(lambda x: x.split() [0] )<categorify>
df_cnt = pd.DataFrame()
for v in range(200):
sr = test_df[f"var_{v}"]
enc = CountEncoder()
enc.fit(sr)
 df_cnt[f"cnt_{v}"] = enc.transform(sr)
 test_df["target"] = -df_cnt.min(1)
 del df_cnt
le = LabelEncoder()
shops['city_id'] = le.fit_transform(shops['city'] )<groupby>
df_merged = merge_train_test(train_df, test_df)
 df_merged.tail()
prices = train.groupby(['item_id','shop_id','date_block_num'],as_index=False ).agg(
 {
'item_price':{'mean_price':'mean','price_std':'std','price_count':'count'},
'item_cnt_day':{'sold_total':'sum'}
}
).fillna(0 )<rename_columns>
%%time
count_enc = [None] * 200
df_real = df_merged[df_merged["target"]!=-2]
for v in range(200):
enc = CountEncoder()
enc.fit(df_real[f"var_{v}"])
 count_enc[v] = enc.transform(df_merged[f"var_{v}"])
 for v in range(200):
df_merged[f"cnt_{v}"] = count_enc[v]
del df_real
prices.columns = ['item_id','shop_id','date_block_num','mean_price','price_std','price_count','sold_total']<feature_engineering>
for v in range(200):
df_merged[f"var_{v}_minmax"] = StandardScaler().fit_transform(df_merged[f"var_{v}"].values.reshape(-1, 1))
df_merged[f"cnt_{v}_minmax"] = MinMaxScaler().fit_transform(df_merged[f"cnt_{v}"].values.reshape(-1, 1))
df_merged.drop(columns=[f"var_{v}" for v in range(200)]+[f"cnt_{v}" for v in range(200)], inplace=True )
prices['date_block_num']+=1<filter>
train_df, test_df = split_train_test(df_merged)
 target = train_df['target']
gc.collect()
print(train_df.shape)
 test_df.head()
shops[(shops.shop_id==55)|(shops.shop_id==12)]<merge>
class Model(nn.Module):
def __init__(self):
super(Model, self ).__init__()
self.hidden_size = 64
self.relu = nn.ReLU()
self.conv1 = nn.Conv1d(2, self.hidden_size, kernel_size=1)
 self.conv2 = nn.Conv1d(self.hidden_size, self.hidden_size*2, kernel_size=1)
 self.conv3 = nn.Conv1d(self.hidden_size*2, self.hidden_size*4, kernel_size=1)
 self.conv4 = nn.Conv1d(self.hidden_size*4, self.hidden_size*8, kernel_size=1)
 self.conv5 = nn.Conv1d(self.hidden_size*8, self.hidden_size*16, kernel_size=1)
 self.conv6 = nn.Conv1d(self.hidden_size*16, self.hidden_size*32, kernel_size=1)
 self.fc = nn.Linear(self.hidden_size*32*200, 2)
 def forward(self, x_):
x = self.conv1(x_)
 x = self.relu(x)
 x = self.conv2(x)
 x = self.relu(x)
 x = self.conv3(x)
 x = self.relu(x)
 x = self.conv4(x)
 x = self.relu(x)
 x = self.conv5(x)
 x = self.relu(x)
 x = self.conv6(x)
 x = self.relu(x)
 x = x.view(x.shape[0], -1)
 x = self.fc(x)
 return x
X.sort_values(by = ['shop_id','item_id','date_block_num'],inplace=True)
 month_ago = 3
for i in range(1, month_ago+1):
colname = 'prev' + str(i)
 X[colname] = X.target.shift(i)
 mask1 = X.shop_id != X.shop_id.shift(i)
 mask2 = X.item_id!= X.item_id.shift(i)
 mask = mask1|mask2
X[colname][mask] = 0
train_columns = ['shop_id','item_id','target']+['prev'+str(i)for i in range(1,month_ago+1)]
test = test.merge(X[X.date_block_num==33][train_columns],on = ['shop_id','item_id'],how = 'left' )<rename_columns>
class TrainData(torch.utils.data.Dataset):
def __init__(self, trn_X, trn_y):
self.trn_X = trn_X
self.trn_y = trn_y
def __len__(self):
return self.trn_X.shape[0]
def __getitem__(self, idx):
return self.trn_X[idx], self.trn_y[idx], idx
def shuffle(self):
trn_X = self.trn_X.to("cpu" ).numpy()
trn_y = self.trn_y.to("cpu" ).numpy()
trn_X_pos = trn_X[trn_y==1].transpose(2,0,1)
 trn_X_neg = trn_X[trn_y==0].transpose(2,0,1)
 for c in trn_X_pos:
np.random.shuffle(c)
 for c in trn_X_neg:
np.random.shuffle(c)
 trn_X[trn_y==1] = trn_X_pos.transpose(1,2,0)
 trn_X[trn_y==0] = trn_X_neg.transpose(1,2,0)
 self.trn_X = torch.from_numpy(trn_X ).to(device)
 class ValidData(torch.utils.data.Dataset):
def __init__(self, val_X, val_y):
self.val_X = val_X
self.val_y = val_y
def __len__(self):
return self.val_X.shape[0]
def __getitem__(self, idx):
return self.val_X[idx], self.val_y[idx], idx
class TestData(torch.utils.data.Dataset):
def __init__(self, test_X):
self.test_X = test_X
def __len__(self):
return self.test_X.shape[0]
def __getitem__(self, idx):
return self.test_X[idx], -1, idx
test.rename(columns = {'prev3':'prev4','prev2':'prev3','prev1':'prev2','target':'prev1'}, inplace = True)
 del test['prev'+str(month_ago+1)] 
<prepare_output>
BATCH_SIZE = 256
EARLY_STOPPING = 20
skf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED_SKF)
 oof = np.zeros(len(train_df))
fold_oof = np.zeros(( N_SPLITS, len(train_df)))
 fold_preds = np.zeros(( N_SPLITS, len(test_df)))
 predictions = np.zeros(len(test_df))
loss_func = nn.CrossEntropyLoss()
for fold_,(trn_idx, val_idx)in enumerate(skf.split(train_df.values, target.values)) :
print("fold n°{}".format(fold_))
features = [f"var_{v}_minmax" for v in range(200)] + [f"cnt_{v}_minmax" for v in range(200)]
trn_X_npy, trn_y_npy = train_df.iloc[trn_idx][features].values.astype(np.float32), target.iloc[trn_idx].values
val_X_npy, val_y_npy = train_df.iloc[val_idx][features].values.astype(np.float32), target.iloc[val_idx].values
trn_X, trn_y = torch.tensor(trn_X_npy.reshape(-1, 2, 200)).to(device), torch.tensor(trn_y_npy ).to(device)
 val_X, val_y = torch.tensor(val_X_npy.reshape(-1, 2, 200)).to(device), torch.tensor(val_y_npy ).to(device)
 test_X = torch.tensor(test_df[features].values.astype(np.float32 ).reshape(-1, 2, 200)).to(device)
 trn_dataset = TrainData(trn_X, trn_y)
 val_dataset = ValidData(val_X, val_y)
 test_dataset = TestData(test_X)
 val_loader = torch.utils.data.DataLoader(dataset=val_dataset, batch_size=256)
 test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=256)
 filename_net = f"net_{fold_}.pth"
net = Model().to(device)
 optimizer = optim.Adam(net.parameters() , lr=0.00002)
 best_epoch = 0
min_auc = 0.5
for epoch in range(100):
if epoch - EARLY_STOPPING > best_epoch:
break
trn_dataset.shuffle()
trn_loader = torch.utils.data.DataLoader(dataset=trn_dataset, batch_size=BATCH_SIZE, shuffle=True)
 net = net.train()
oof_ = np.zeros(( len(trn_idx), 2), dtype=np.float32)
 for data, label, idx in trn_loader:
optimizer.zero_grad()
output = net(data)
 loss = loss_func(output, label)
 loss.backward()
oof_[idx.numpy() ] = output.detach().cpu().numpy()
optimizer.step()
net = net.eval()
with torch.no_grad() :
loss = loss_func(torch.from_numpy(oof_), torch.from_numpy(trn_y_npy))
auc = roc_auc_score(trn_y_npy, oof_[:, 1] - oof_[:, 0])
 print(f"epoch {epoch}: train loss: {loss:.5f}, train auc: {auc:.5f}, ", end="")
 output = np.zeros(( len(val_idx), 2), dtype=np.float32)
 for data, _, idx in val_loader:
output[idx.numpy() ] = net(data ).detach().cpu().numpy()
loss = loss_func(torch.from_numpy(output), torch.from_numpy(val_y_npy))
auc = roc_auc_score(val_y_npy, output[:, 1] - output[:, 0])
 print(f"valid loss: {loss:.5f}, valid auc: {auc:.5f}")
 if auc > min_auc:
torch.save(net.state_dict() , filename_net)
 min_auc = auc
best_epoch = epoch
net.load_state_dict(torch.load(filename_net))
output = np.zeros(( len(val_idx), 2), dtype=np.float32)
 for data, _, idx in val_loader:
output[idx.numpy() ] = net(data ).detach().cpu().numpy()
val_auc = roc_auc_score(val_y_npy, output[:, 1] - output[:, 0])
 print(f"fold {fold_} auc: {val_auc:.5f}")
 oof[val_idx] = expit(output[:, 1] - output[:, 0])
 fold_oof[fold_, val_idx] = oof[val_idx]
output = np.zeros(( len(test_dataset), 2), dtype=np.float32)
 for data, _, idx in test_loader:
output[idx.numpy() ] = net(data ).detach().cpu().numpy()
fold_preds[fold_, :] = expit(output[:, 1] - output[:, 0])
 predictions += fold_preds[fold_] / N_SPLITS
break
np.save("oof.npy", oof)
 np.save("fold_oof.npy", fold_oof)
 np.save("fold_preds.npy", fold_preds)
 np.save("predictions.npy", predictions)
 print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )
<drop_column><EOS> 
sub_df = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df["target"] = predictions
sub_df.to_csv("submission.csv", index=False )