Kaggle competition: BU CS506 Spring 2020 Midterm
<SOS> metric: rmse Kaggle data source: bu-cs506-spring-2020-midterm<save_to_csv>
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
preds_oof_final = preds_nn_oof
preds_test_final = preds_nn_test
preds_fake_final = preds_nn_fake
print('oof : ', roc_auc_score(target_train, preds_oof_final))
if use_experimental:
print('test : ', roc_auc_score(target_test, preds_test_final))
print('train: ', roc_auc_score(target_fake, preds_fake_final))
if not use_experimental:
sub = pd.DataFrame({"ID_code": test_df.ID_code.values})
 predictions_all = np.zeros(test_df.shape[0])
 predictions_all[indices_real] = preds_test_final
predictions_all[indices_fake] = preds_fake_final
sub["target"] = predictions_all
sub.to_csv("submission.csv", index=False)
 print(sub.head(20))<feature_engineering>
data = pd.read_csv('/kaggle/input/bu-cs506-spring-2020-midterm/train.csv' )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 features = [x for x in train_df.columns if x.startswith("var")]
hist_df = pd.DataFrame()
for var in features:
var_stats = train_df[var].append(test_df[var] ).value_counts()
hist_df[var] = pd.Series(test_df[var] ).map(var_stats)
 hist_df[var] = hist_df[var] > 1
ind = hist_df.sum(axis=1)!= 200
var_stats = {var:train_df[var].append(test_df[ind][var] ).value_counts() for var in features}
pred = 0
for var in features:
model = lgb.LGBMClassifier(**{
'learning_rate':0.05, 'max_bin': 165, 'max_depth': 5, 'min_child_samples': 150,
'min_child_weight': 0.1, 'min_split_gain': 0.0018, 'n_estimators': 41,
'num_leaves': 6, 'reg_alpha': 2.0, 'reg_lambda': 2.54, 'objective': 'binary', 'n_jobs': -1})
 model = model.fit(np.hstack([train_df[var].values.reshape(-1,1),
train_df[var].map(var_stats[var] ).values.reshape(-1,1)]),
train_df["target"].values)
 pred += logit(model.predict_proba(np.hstack([test_df[var].values.reshape(-1,1),
test_df[var].map(var_stats[var] ).values.reshape(-1,1)])) [:,1])
 pd.DataFrame({"ID_code":test_df["ID_code"], "target":pred} ).to_csv("submission.csv", index=False)
<feature_engineering>
print('There are', str(data.shape[0]), 'records in total.' )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 features = [x for x in train_df.columns if x.startswith("var")]
var_stats = {var:train_df[var].value_counts() for var in features} 
<train_model>
HelpfulnessNumerator0 = data[data['HelpfulnessNumerator'] == 0]['HelpfulnessNumerator'].value_counts()
HelpfulnessNumerator1 = data[data['HelpfulnessNumerator'] == 1]['HelpfulnessNumerator'].value_counts()
HelpfulnessNumerator2 = data[data['HelpfulnessNumerator'] == 2]['HelpfulnessNumerator'].value_counts()
HelpfulnessNumerator3 = data[data['HelpfulnessNumerator'] == 3]['HelpfulnessNumerator'].value_counts()
HelpfulnessNumeratorMoreThan3 = data[data['HelpfulnessNumerator'] > 3]['HelpfulnessNumerator'].value_counts()
labels = '0', '1', '2', '3', 'more than 3'
sizes = [HelpfulnessNumerator0.values.item() , HelpfulnessNumerator1.values.item() , HelpfulnessNumerator2.values.item() ,
HelpfulnessNumerator3.values.item() , HelpfulnessNumeratorMoreThan3.values.sum() ]
explode =(0, 0, 0, 0, 0)
 fig, ax = plt.subplots()
ax.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',
shadow=True, startangle=90)
 ax.axis('equal')
 plt.title(' Portions of Amount of Helpfulness Labels')
 plt.show()
pred = 0
for var in features:
model = lgb.LGBMClassifier(**{
'learning_rate':0.05, 'max_bin': 165, 'max_depth': 5, 'min_child_samples': 150,
'min_child_weight': 0.1, 'min_split_gain': 0.0018, 'n_estimators': 41,
'num_leaves': 6, 'reg_alpha': 2.0, 'reg_lambda': 2.54, 'objective': 'binary', 'n_jobs': -1})
 model = model.fit(np.hstack([train_df[var].values.reshape(-1,1),
train_df[var].map(var_stats[var] ).values.reshape(-1,1)]),
train_df["target"].values)
 X_test = np.hstack([test_df[var].values.reshape(-1,1),
test_df[var].map(var_stats[var] ).values.reshape(-1,1)])
 X_test[:,1] = X_test[:,1] + 1
X_test[:,1][np.isnan(X_test[:,1])] = 1
pred += logit(model.predict_proba(X_test)[:,1])
 pd.DataFrame({"ID_code":test_df["ID_code"], "target":pred} ).to_csv("submission.csv", index=False )<set_options>
HelpfulnessDenominator0 = data[data['HelpfulnessDenominator'] == 0]['HelpfulnessDenominator'].value_counts()
HelpfulnessDenominator1 = data[data['HelpfulnessDenominator'] == 1]['HelpfulnessDenominator'].value_counts()
HelpfulnessDenominator2 = data[data['HelpfulnessDenominator'] == 2]['HelpfulnessDenominator'].value_counts()
HelpfulnessDenominator3 = data[data['HelpfulnessDenominator'] == 3]['HelpfulnessDenominator'].value_counts()
HelpfulnessDenominatorMoreThan3 = data[data['HelpfulnessDenominator'] > 3]['HelpfulnessDenominator'].value_counts()
labels = '0', '1', '2', '3', 'more than 3'
sizes = [HelpfulnessDenominator0.values.item() , HelpfulnessDenominator1.values.item() , HelpfulnessDenominator2.values.item() ,
HelpfulnessDenominator3.values.item() , HelpfulnessDenominatorMoreThan3.values.sum() ]
explode =(0, 0, 0, 0, 0)
 fig, ax = plt.subplots()
ax.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',
shadow=True, startangle=90)
 ax.axis('equal')
 plt.title(' Portions of Amount of Comments Watched')
 plt.show()
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 print(device)
 warnings.filterwarnings('ignore')
 PATH=".. /input/"
N_SPLITS = 10
SEED_SKF = 4221 
<split>
OHE = OneHotEncoder(sparse=True)
 ID_fitter = OHE.fit(data[['ProductId', 'UserId']])
 IDs = ID_fitter.transform(data[['ProductId', 'UserId']] )
def merge_train_test(df_train, df_test):
if "target" not in df_test.columns.values:
df_test["target"] = -1
res = pd.concat([df_train, df_test])
 res.reset_index(inplace=True, drop=True)
 return res
def split_train_test(df):
df_train = df[df["target"] >= 0]
df_test = df[df["target"] <= -1]
df_train.reset_index(inplace=True, drop=True)
 df_test.reset_index(inplace=True, drop=True)
 assert list(df_train["ID_code"].values)== [f"train_{i}" for i in range(200000)]
assert list(df_test["ID_code"].values)== [f"test_{i}" for i in range(200000)]
return df_train, df_test<load_from_csv>
data['Text'].loc[data['Text'].isna() ] = ''
data['Summary'].loc[data['Summary'].isna() ] = ''
%%time
train_df = pd.read_csv(PATH+"train.csv")
 test_df = pd.read_csv(PATH+"test.csv" )<categorify>
data['Helpful'] = data['HelpfulnessNumerator']
data['Unhelpful'] = data['HelpfulnessDenominator'] - data['HelpfulnessNumerator']
scaler = StandardScaler()
scalerFitter = scaler.fit(data[['Helpful', 'Unhelpful', 'Time']])
 data[['Helpful', 'Unhelpful', 'Time']] = scalerFitter.transform(data[['Helpful', 'Unhelpful', 'Time']])
 data = data.drop(['HelpfulnessDenominator','HelpfulnessNumerator'], axis=1 )
class CountEncoder:
def fit(self, series):
self.counts = series.groupby(series ).count()
def transform(self, series):
return series.map(self.counts ).fillna(0 ).astype(np.int16 )<categorify>
text_vectorizer = TfidfVectorizer(input='content', analyzer='word', stop_words='english')
 summary_vectorizer = TfidfVectorizer(input='content', analyzer='word', stop_words='english')
 text_fitter = text_vectorizer.fit(data['Text'])
 text_matrix = text_fitter.transform(data['Text'])
 summary_fitter = summary_vectorizer.fit(data['Summary'])
 summary_matrix = summary_fitter.transform(data['Summary'] )
df_cnt = pd.DataFrame()
for v in range(200):
sr = test_df[f"var_{v}"]
enc = CountEncoder()
enc.fit(sr)
 df_cnt[f"cnt_{v}"] = enc.transform(sr)
 test_df["target"] = -df_cnt.min(1)
 del df_cnt<merge>
text_matrix, summary_matrix
df_merged = merge_train_test(train_df, test_df)
 df_merged.tail()<categorify>
numerical = scipy.sparse.csr_matrix(data[['Helpful', 'Unhelpful', 'Time']].values )
%%time
count_enc = [None] * 200
df_real = df_merged[df_merged["target"]!=-2]
for v in range(200):
enc = CountEncoder()
enc.fit(df_real[f"var_{v}"])
 count_enc[v] = enc.transform(df_merged[f"var_{v}"])
 for v in range(200):
df_merged[f"cnt_{v}"] = count_enc[v]
del df_real<normalization>
X = hstack([text_matrix, summary_matrix, numerical, IDs] )
for v in range(200):
df_merged[f"var_{v}_minmax"] = StandardScaler().fit_transform(df_merged[f"var_{v}"].values.reshape(-1, 1))
df_merged[f"cnt_{v}_minmax"] = MinMaxScaler().fit_transform(df_merged[f"cnt_{v}"].values.reshape(-1, 1))
df_merged.drop(columns=[f"var_{v}" for v in range(200)]+[f"cnt_{v}" for v in range(200)], inplace=True )<split>
mask = data["Score"].isnull()
ind_test = mask.to_numpy().nonzero() [0]
ind_train =(~ mask ).to_numpy().nonzero() [0]
train_X = scipy.sparse.csr_matrix(X)[ind_train]
test_X = scipy.sparse.csr_matrix(X)[ind_test]
train_df, test_df = split_train_test(df_merged)
 target = train_df['target']
gc.collect()
print(train_df.shape)
 test_df.head()<choose_model_class>
train_Y = data['Score'].loc[data['Score'].isna() == False]
test_Y = data['Score'].loc[data['Score'].isna() ]
train_Y = train_Y.reset_index() ['Score']
test_Y = test_Y.reset_index() ['Score']
class Model(nn.Module):
def __init__(self):
super(Model, self ).__init__()
self.hidden_size = 64
self.relu = nn.ReLU()
self.conv1 = nn.Conv1d(2, self.hidden_size, kernel_size=1)
 self.conv2 = nn.Conv1d(self.hidden_size, self.hidden_size*2, kernel_size=1)
 self.conv3 = nn.Conv1d(self.hidden_size*2, self.hidden_size*4, kernel_size=1)
 self.conv4 = nn.Conv1d(self.hidden_size*4, self.hidden_size*8, kernel_size=1)
 self.conv5 = nn.Conv1d(self.hidden_size*8, self.hidden_size*16, kernel_size=1)
 self.conv6 = nn.Conv1d(self.hidden_size*16, self.hidden_size*32, kernel_size=1)
 self.fc = nn.Linear(self.hidden_size*32*200, 2)
 def forward(self, x_):
x = self.conv1(x_)
 x = self.relu(x)
 x = self.conv2(x)
 x = self.relu(x)
 x = self.conv3(x)
 x = self.relu(x)
 x = self.conv4(x)
 x = self.relu(x)
 x = self.conv5(x)
 x = self.relu(x)
 x = self.conv6(x)
 x = self.relu(x)
 x = x.view(x.shape[0], -1)
 x = self.fc(x)
 return x<categorify>
ros = RandomOverSampler(random_state=42)
 train_X, train_Y = ros.fit_resample(train_X, train_Y )
class TrainData(torch.utils.data.Dataset):
def __init__(self, trn_X, trn_y):
self.trn_X = trn_X
self.trn_y = trn_y
def __len__(self):
return self.trn_X.shape[0]
def __getitem__(self, idx):
return self.trn_X[idx], self.trn_y[idx], idx
def shuffle(self):
trn_X = self.trn_X.to("cpu" ).numpy()
trn_y = self.trn_y.to("cpu" ).numpy()
trn_X_pos = trn_X[trn_y==1].transpose(2,0,1)
 trn_X_neg = trn_X[trn_y==0].transpose(2,0,1)
 for c in trn_X_pos:
np.random.shuffle(c)
 for c in trn_X_neg:
np.random.shuffle(c)
 trn_X[trn_y==1] = trn_X_pos.transpose(1,2,0)
 trn_X[trn_y==0] = trn_X_neg.transpose(1,2,0)
 self.trn_X = torch.from_numpy(trn_X ).to(device)
 class ValidData(torch.utils.data.Dataset):
def __init__(self, val_X, val_y):
self.val_X = val_X
self.val_y = val_y
def __len__(self):
return self.val_X.shape[0]
def __getitem__(self, idx):
return self.val_X[idx], self.val_y[idx], idx
class TestData(torch.utils.data.Dataset):
def __init__(self, test_X):
self.test_X = test_X
def __len__(self):
return self.test_X.shape[0]
def __getitem__(self, idx):
return self.test_X[idx], -1, idx 
<init_hyperparams>
X_train, X_valid, y_train, y_valid = train_test_split(train_X, train_Y, test_size=0.2, random_state=42 )
BATCH_SIZE = 256
EARLY_STOPPING = 20
skf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED_SKF)
 oof = np.zeros(len(train_df))
fold_oof = np.zeros(( N_SPLITS, len(train_df)))
 fold_preds = np.zeros(( N_SPLITS, len(test_df)))
 predictions = np.zeros(len(test_df))
loss_func = nn.CrossEntropyLoss()
for fold_,(trn_idx, val_idx)in enumerate(skf.split(train_df.values, target.values)) :
print("fold n°{}".format(fold_))
features = [f"var_{v}_minmax" for v in range(200)] + [f"cnt_{v}_minmax" for v in range(200)]
trn_X_npy, trn_y_npy = train_df.iloc[trn_idx][features].values.astype(np.float32), target.iloc[trn_idx].values
val_X_npy, val_y_npy = train_df.iloc[val_idx][features].values.astype(np.float32), target.iloc[val_idx].values
trn_X, trn_y = torch.tensor(trn_X_npy.reshape(-1, 2, 200)).to(device), torch.tensor(trn_y_npy ).to(device)
 val_X, val_y = torch.tensor(val_X_npy.reshape(-1, 2, 200)).to(device), torch.tensor(val_y_npy ).to(device)
 test_X = torch.tensor(test_df[features].values.astype(np.float32 ).reshape(-1, 2, 200)).to(device)
 trn_dataset = TrainData(trn_X, trn_y)
 val_dataset = ValidData(val_X, val_y)
 test_dataset = TestData(test_X)
 val_loader = torch.utils.data.DataLoader(dataset=val_dataset, batch_size=256)
 test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=256)
 filename_net = f"net_{fold_}.pth"
net = Model().to(device)
 optimizer = optim.Adam(net.parameters() , lr=0.00002)
 best_epoch = 0
min_auc = 0.5
for epoch in range(100):
if epoch - EARLY_STOPPING > best_epoch:
break
trn_dataset.shuffle()
trn_loader = torch.utils.data.DataLoader(dataset=trn_dataset, batch_size=BATCH_SIZE, shuffle=True)
 net = net.train()
oof_ = np.zeros(( len(trn_idx), 2), dtype=np.float32)
 for data, label, idx in trn_loader:
optimizer.zero_grad()
output = net(data)
 loss = loss_func(output, label)
 loss.backward()
oof_[idx.numpy() ] = output.detach().cpu().numpy()
optimizer.step()
net = net.eval()
with torch.no_grad() :
loss = loss_func(torch.from_numpy(oof_), torch.from_numpy(trn_y_npy))
auc = roc_auc_score(trn_y_npy, oof_[:, 1] - oof_[:, 0])
 print(f"epoch {epoch}: train loss: {loss:.5f}, train auc: {auc:.5f}, ", end="")
 output = np.zeros(( len(val_idx), 2), dtype=np.float32)
 for data, _, idx in val_loader:
output[idx.numpy() ] = net(data ).detach().cpu().numpy()
loss = loss_func(torch.from_numpy(output), torch.from_numpy(val_y_npy))
auc = roc_auc_score(val_y_npy, output[:, 1] - output[:, 0])
 print(f"valid loss: {loss:.5f}, valid auc: {auc:.5f}")
 if auc > min_auc:
torch.save(net.state_dict() , filename_net)
 min_auc = auc
best_epoch = epoch
net.load_state_dict(torch.load(filename_net))
output = np.zeros(( len(val_idx), 2), dtype=np.float32)
 for data, _, idx in val_loader:
output[idx.numpy() ] = net(data ).detach().cpu().numpy()
val_auc = roc_auc_score(val_y_npy, output[:, 1] - output[:, 0])
 print(f"fold {fold_} auc: {val_auc:.5f}")
 oof[val_idx] = expit(output[:, 1] - output[:, 0])
 fold_oof[fold_, val_idx] = oof[val_idx]
output = np.zeros(( len(test_dataset), 2), dtype=np.float32)
 for data, _, idx in test_loader:
output[idx.numpy() ] = net(data ).detach().cpu().numpy()
fold_preds[fold_, :] = expit(output[:, 1] - output[:, 0])
 predictions += fold_preds[fold_] / N_SPLITS
break
np.save("oof.npy", oof)
 np.save("fold_oof.npy", fold_oof)
 np.save("fold_preds.npy", fold_preds)
 np.save("predictions.npy", predictions)
 print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )<save_to_csv>
def CVKFold(k, X, y, model):
np.random.seed(1)
 highest_accuracy = float('inf')
 best_model = None
kf = KFold(n_splits = k,shuffle =True)
 for train_index,test_index in kf.split(X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
model.fit(X_train,y_train)
 y_test_pred = model.predict(X_test)
 test_accuracy = mean_squared_error(y_test_pred, y_test)
 print("The accuracy is " + str(test_accuracy))
if test_accuracy < highest_accuracy:
best_model = model
highest_accuracy = test_accuracy
print("The highest accuracy is " + str(highest_accuracy))
return best_model, highest_accuracy
sub_df = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df["target"] = predictions
sub_df.to_csv("submission.csv", index=False )<load_from_csv>
model = LogisticRegression(random_state = 0)
 model = model.fit(train_X, train_Y)
<save_to_csv><EOS> 
sample = pd.read_csv('/kaggle/input/bu-cs506-spring-2020-midterm/sample.csv')
 predict_df = pd.DataFrame(sample)
 predict_df['Score'] = model.predict(test_X)
 predict_df.to_csv(r'sample.csv',index=False )