Kaggle competition: DeepNLP HSE Course
def t_encoding(tr,va,by):
df = tr.groupby(by ).agg({'target':['sum','count']})
 cols = ['sum_y','count_y']
df.columns = cols
df = df.reset_index()
df = df.sort_values(by)
 df['r'] = df['sum_y'].cumsum() / df['count_y'].cumsum()
df.drop(['sum_y','count_y'],axis=1,inplace=True)
 return va.merge(df,on=by,how='left')['r'].values 
<categorify>
train = pd.read_csv('/kaggle/input/deepnlp-hse-course/train.csv')
 test = pd.read_csv('/kaggle/input/deepnlp-hse-course/test.csv' )
var_cols = features
for col in tqdm(var_cols):
te_r = t_encoding(train_df,test_df,col)
 test_df.loc[:,col+'_r'] = te_r
folds = StratifiedKFold(n_splits=12, shuffle=False, random_state=99999)
 for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values, target.values)) :
print("Fold {}".format(fold_))
tr = train_df.loc[trn_idx,var_cols+['target']]
va = train_df.loc[val_idx,var_cols+['target']]
for col in tqdm(var_cols):
encoded = t_encoding(tr,va,col)
 train_df.loc[val_idx,col+'_r'] = encoded
features = [col for col in train_df.columns if('var' in col)]
train_df = train_df[features].reset_index(drop=True)
 print('data prepared: {}'.format(train_df.shape))<init_hyperparams>
import torch
from torch.utils.data import Dataset, DataLoader
from nltk.tokenize import word_tokenize, wordpunct_tokenize
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
param = {
'bagging_freq': 5,
'bagging_fraction': 0.38, 'boost_from_average':'false',
'boost': 'gbdt', 'feature_fraction': 1, 'learning_rate': 0.0085,
'max_depth': -1, 'metric':'auc', 'min_data_in_leaf': 80, 'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13, 'num_threads': 8, 'tree_learner': 'serial', 'objective': 'binary',
'reg_alpha': 0.1302650970728192, 'reg_lambda': 0.3603427518866501,'verbosity': 1
} 
<train_model>
def process_text(text):
words = wordpunct_tokenize(text.lower())
 return words
folds = StratifiedKFold(n_splits=12, shuffle=False, random_state=99999)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_df.values, target.values)) :
print("Fold {}".format(fold_))
trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx])
 clf = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 2000)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)
 predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)/ folds.n_splits
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)))
 sub = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub["target"] = predictions
sub.to_csv("submission.csv", index=False )<load_from_csv>
word2freq = {}
for question in tqdm(train.question):
words = process_text(question)
 for word in words:
if word in word2freq:
word2freq[word] += 1
else:
word2freq[word] = 1
warnings.filterwarnings("ignore")
 test_path = '.. /input/test.csv'
df_test = pd.read_csv(test_path)
 df_test.drop(['ID_code'], axis=1, inplace=True)
 df_test = df_test.values
unique_samples = []
unique_count = np.zeros_like(df_test)
 for feature in range(df_test.shape[1]):
_, index_, count_ = np.unique(df_test[:, feature], return_counts=True, return_index=True)
 unique_count[index_[count_ == 1], feature] += 1
real_samples_indexes = np.argwhere(np.sum(unique_count, axis=1)> 0)[:, 0]
synthetic_samples_indexes = np.argwhere(np.sum(unique_count, axis=1)== 0)[:, 0]
print('Found',len(real_samples_indexes),'real test')
 print('Found',len(synthetic_samples_indexes),'fake test')
 d = {}
for i in range(200): d['var_'+str(i)] = 'float32'
d['target'] = 'uint8'
d['ID_code'] = 'object'
train = pd.read_csv('.. /input/train.csv', dtype=d)
 test = pd.read_csv('.. /input/test.csv', dtype=d)
 print('Loaded',len(train),'rows of train')
 print('Loaded',len(test),'rows of test' )<data_type_conversions>
word2index = {'PAD': 0}
vectors = []
word2vec_file = open('/kaggle/input/fasttest-common-crawl-russian/cc.ru.300.vec')
 n_words, embedding_dim = word2vec_file.readline().split()
n_words, embedding_dim = int(n_words), int(embedding_dim)
 vectors.append(np.zeros(( 1, embedding_dim)))
 progress_bar = tqdm(desc='Read word2vec', total=n_words)
 while True:
line = word2vec_file.readline().strip()
if not line:
break
current_parts = line.split()
current_word = ' '.join(current_parts[:-embedding_dim])
 if current_word in word2freq:
word2index[current_word] = len(word2index)
 current_vectors = current_parts[-embedding_dim:]
current_vectors = np.array(list(map(float, current_vectors)))
 current_vectors = np.expand_dims(current_vectors, 0)
 vectors.append(current_vectors)
 progress_bar.update(1)
 progress_bar.close()
word2vec_file.close()
vectors = np.concatenate(vectors )
def encode_FE(df,col,test):
cv = df[col].value_counts()
nm = col+'_FE'
df[nm] = df[col].map(cv)
 test[nm] = test[col].map(cv)
 test[nm].fillna(0,inplace=True)
 if cv.max() <=255:
df[nm] = df[nm].astype('uint8')
 test[nm] = test[nm].astype('uint8')
 else:
df[nm] = df[nm].astype('uint16')
 test[nm] = test[nm].astype('uint16')
 return
test['target'] = -1
comb = pd.concat([train,test.loc[real_samples_indexes]],axis=0,sort=True)
 for i in range(200): encode_FE(comb,'var_'+str(i),test)
 train = comb[:len(train)]; del comb
print('Added 200 new magic features!' )<define_variables>
unk_words = [word for word in word2freq if word not in word2index]
unk_counts = [word2freq[word] for word in unk_words]
n_unk = sum(unk_counts)* 100 / sum(list(word2freq.values()))
sub_sample_unk_words = {word: word2freq[word] for word in unk_words}
sorted_unk_words = list(sorted(sub_sample_unk_words, key=lambda x: sub_sample_unk_words[x], reverse=True))
print('Мы не знаем {:.2f} % слов в датасете'.format(n_unk))
print('Количество неизвестных слов {} из {}, то есть {:.2f} % уникальных слов в словаре'.format(
 len(unk_words), len(word2freq), len(unk_words)* 100 / len(word2freq)))
 print('В среднем каждое встречается {:.2f} раз'.format(np.mean(unk_counts)))
 print()
print('Топ 5 невошедших слов:')
 for i in range(5):
print(sorted_unk_words[i], 'с количеством вхождениий -', word2freq[sorted_unk_words[i]] )
param = {
'learning_rate': 0.04,
'num_leaves': 3,
'metric':'auc',
'boost_from_average':'false',
'feature_fraction': 1.0,
'max_depth': -1,
'objective': 'binary',
'verbosity': -10}
train2 = train.sample(frac=1,random_state=42)
 evals_result = {}
num_vars = 200
all_oof = np.zeros(( len(train2),num_vars+1))
all_oof[:,0] = np.ones(len(train2))
all_oofB = np.zeros(( len(train2),num_vars+1))
all_oofB[:,0] = np.ones(len(train2))
all_preds = np.zeros(( len(test),num_vars+1))
all_preds[:,0] = np.ones(len(test))
all_predsB = np.zeros(( len(test),num_vars+1))
all_predsB[:,0] = np.ones(len(test))
for j in range(num_vars):
features = ['var_'+str(j),'var_'+str(j)+'_FE']
oof = np.zeros(len(train2))
preds = np.zeros(len(test))
plt.figure(figsize=(16,5))
plt.subplot(1,2,2)
 sns.distplot(train2[train2['target']==0]['var_'+str(j)], label = 't=0')
 sns.distplot(train2[train2['target']==1]['var_'+str(j)], label = 't=1')
 plt.legend()
plt.yticks([])
 plt.xlabel('Var_'+str(j))
mn,mx = plt.xlim()
mnFE = train2['var_'+str(j)+'_FE'].min()
mxFE = train2['var_'+str(j)+'_FE'].max()
step = 50
stepB = train2['var_'+str(j)+'_FE'].nunique()
w =(mx-mn)/step
x = w *(np.arange(0,step)+0.5)+ mn
x2 = np.array([])
 for i in range(stepB):
x2 = np.concatenate([x,x2])
 df = pd.DataFrame({'var_'+str(j):x2})
 df['var_'+str(j)+'_FE'] = mnFE +(mxFE-mnFE)/(stepB-1)*(df.index//step)
 df['pred'] = 0
for k in range(5):
valid = train2.iloc[k*40000:(k+1)*40000]
train = train2[ ~train2.index.isin(valid.index)]
trn_data = lgb.Dataset(train[features], label=train['target'])
 val_data = lgb.Dataset(valid[features], label=valid['target'])
 model = lgb.train(param, trn_data, 750, valid_sets = [trn_data, val_data],
verbose_eval=False, evals_result=evals_result)
 x = evals_result['valid_1']['auc']
best = x.index(max(x))
oof[k*40000:(k+1)*40000] = model.predict(valid[features], num_iteration=best)
 preds += model.predict(test[features], num_iteration=best)/5.0
df['pred'] += model.predict(df[features], num_iteration=best)/5.0
val_auc = roc_auc_score(train2['target'],oof)
 print('VAR_'+str(j)+' with magic val_auc =',round(val_auc,5))
all_oof[:,j+1] = oof
all_preds[:,j+1] = preds
x = df['pred'].values
x = np.reshape(x,(stepB,step))
x = np.flip(x,axis=0)
 plt.subplot(1,2,1)
 sns.heatmap(x, cmap='RdBu_r', center=0.0)
 plt.title('VAR_'+str(j)+' Predictions with Magic',fontsize=16)
 plt.xticks(np.linspace(0,49,5),np.round(np.linspace(mn,mx,5),1))
plt.xlabel('Var_'+str(j))
s = min(mxFE-mnFE+1,20)
 plt.yticks(np.linspace(mnFE,mxFE,s)-0.5,np.linspace(mxFE,mnFE,s ).astype('int'))
plt.ylabel('Count')
 plt.show()
features = ['var_'+str(j)]
oof = np.zeros(len(train2))
preds = np.zeros(len(test))
plt.figure(figsize=(16,5))
plt.subplot(1,2,2)
 sns.distplot(train2[train2['target']==0]['var_'+str(j)], label = 't=0')
 sns.distplot(train2[train2['target']==1]['var_'+str(j)], label = 't=1')
 plt.legend()
plt.yticks([])
 plt.xlabel('Var_'+str(j))
mn,mx = plt.xlim()
mnFE = train2['var_'+str(j)+'_FE'].min()
mxFE = train2['var_'+str(j)+'_FE'].max()
step = 50
stepB = train2['var_'+str(j)+'_FE'].nunique()
w =(mx-mn)/step
x = w *(np.arange(0,step)+0.5)+ mn
x2 = np.array([])
 for i in range(stepB):
x2 = np.concatenate([x,x2])
 df = pd.DataFrame({'var_'+str(j):x2})
 df['var_'+str(j)+'_FE'] = mnFE +(mxFE-mnFE)/(stepB-1)*(df.index//step)
 df['pred'] = 0
for k in range(5):
valid = train2.iloc[k*40000:(k+1)*40000]
train = train2[ ~train2.index.isin(valid.index)]
trn_data = lgb.Dataset(train[features], label=train['target'])
 val_data = lgb.Dataset(valid[features], label=valid['target'])
 model = lgb.train(param, trn_data, 750, valid_sets = [trn_data, val_data],
verbose_eval=False, evals_result=evals_result)
 x = evals_result['valid_1']['auc']
best = x.index(max(x))
oof[k*40000:(k+1)*40000] = model.predict(valid[features], num_iteration=best)
 preds += model.predict(test[features], num_iteration=best)/5.0
df['pred'] += model.predict(df[features], num_iteration=best)/5.0
val_auc = roc_auc_score(train2['target'],oof)
 print('VAR_'+str(j)+' without magic val_auc =',round(val_auc,5))
all_oofB[:,j+1] = oof
all_predsB[:,j+1] = preds
x = df['pred'].values
x = np.reshape(x,(stepB,step))
x = np.flip(x,axis=0)
 plt.subplot(1,2,1)
 sns.heatmap(x, cmap='RdBu_r', center=0.0)
 plt.title('VAR_'+str(j)+' Predictions without Magic',fontsize=16)
 plt.xticks(np.linspace(0,49,5),np.round(np.linspace(mn,mx,5),1))
plt.xlabel('Var_'+str(j))
plt.yticks([])
 plt.ylabel('')
 plt.show()<compute_train_metric>
class WordData(Dataset):
def __init__(self, x_data, y_data, word2index, sequence_length=32, pad_token='PAD', verbose=True):
super().__init__()
self.x_data = []
self.y_data = y_data
self.word2index = word2index
self.sequence_length = sequence_length
self.pad_token = pad_token
self.pad_index = self.word2index[self.pad_token]
self.load(x_data, verbose=verbose)
 @staticmethod
def process_text(text):
words = wordpunct_tokenize(text.lower())
 return words
def load(self, data, verbose=True):
data_iterator = tqdm(data, desc='Loading data', disable=not verbose)
 for text in data_iterator:
words = self.process_text(text)
 indexed_words = self.indexing(words)
 self.x_data.append(indexed_words)
 def indexing(self, tokenized_text):
return [self.word2index[token] for token in tokenized_text if token in self.word2index ]
def padding(self, sequence):
if len(sequence)> self.sequence_length:
sequence = sequence[:self.sequence_length]
elif len(sequence)< self.sequence_length:
sequence = sequence + [self.pad_index] *(self.sequence_length - len(sequence))
return sequence
def __len__(self):
return len(self.x_data)
 def __getitem__(self, idx):
x = self.x_data[idx]
x = self.padding(x)
 x = torch.Tensor(x ).long()
y = self.y_data[idx]
return x, y
logrB = sm.Logit(train2['target'], all_oofB[:,:num_vars+1])
 logrB = logrB.fit(disp=0)
 ensemble_predsB = logrB.predict(all_oofB[:,:num_vars+1])
 ensemble_aucB = roc_auc_score(train2['target'],ensemble_predsB)
 print('
print('Combined Model without magic Val_AUC=',round(ensemble_aucB,5))
print()
logr = sm.Logit(train2['target'], all_oof[:,:num_vars+1])
 logr = logr.fit(disp=0)
 ensemble_preds = logr.predict(all_oof[:,:num_vars+1])
 ensemble_auc = roc_auc_score(train2['target'],ensemble_preds)
 print('
print('Combined Model with magic Val_AUC=',round(ensemble_auc,5))
print()<save_to_csv>
x_train, x_validation, y_train, y_validation = train_test_split(train.question, train.main_category, test_size=0.1)
 train_dataset = WordData(list(x_train), list(y_train), word2index)
 train_loader = DataLoader(train_dataset, batch_size=64)
 validation_dataset = WordData(list(x_validation), list(y_validation), word2index)
 validation_loader = DataLoader(validation_dataset, batch_size=64)
 test_dataset = WordData(list(test.question), np.zeros(( test.shape[0])) , word2index)
 test_loader = DataLoader(test_dataset, batch_size=64 )
print('Test predictions saved as submission.csv')
 print('OOF predictions saved as oof_submission.csv')
 print('Histogram of test predictions displayed below:')
 sub = train2[['ID_code','target']].copy()
sub['predict'] = ensemble_preds
sub.reset_index(inplace=True)
 sub.sort_values('index',inplace=True)
 sub.to_csv('oof_submission.csv',index=False)
 test_preds = logr.predict(all_preds[:,:num_vars+1])
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = test_preds
sub.to_csv('submission.csv',index=False)
 b = plt.hist(sub['target'], bins=200 )<set_options>
for x, y in test_loader:
break
warnings.filterwarnings('ignore')
 np.random.seed(42 )<load_from_csv>
class DeepAverageNetwork(torch.nn.Module):
def __init__(self, embedding_matrix, n_classes):
super().__init__()
self.embedding_layer = torch.nn.Embedding.from_pretrained(torch.Tensor(embedding_matrix))
self.layers = torch.nn.Sequential(torch.nn.Linear(300, 256),
torch.nn.ReLU() ,
torch.nn.Linear(256, 128),
torch.nn.ReLU() ,
torch.nn.Linear(128, n_classes))
def forward(self, x):
sequence_lengths =(x > 0 ).sum(dim=1)
 sequence_lengths[sequence_lengths == 0.] = 1
x = self.embedding_layer(x)
 x = x.mean(dim=-2)
 lengths_scaling = sequence_lengths.float() / x.size(1)
 lengths_scaling = lengths_scaling.unsqueeze(1 ).repeat(( 1, x.size(-1)))
 x /= lengths_scaling.to(x.device)
 x = self.layers(x)
 return x
use_experimental = False
train_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/train.csv')
 test_df = pd.read_csv('.. /input/santander-customer-transaction-prediction/test.csv')
 indices_fake = np.load('.. /input/list-of-fake-samples-and-public-private-lb-split/synthetic_samples_indexes.npy')
 indices_pub = np.load('.. /input/list-of-fake-samples-and-public-private-lb-split/public_LB.npy')
 indices_pri = np.load('.. /input/list-of-fake-samples-and-public-private-lb-split/private_LB.npy')
 indices_real = np.concatenate([indices_pub, indices_pri])
 features = [c for c in train_df.columns if c not in ['ID_code', 'target']]
target_train = train_df['target']
X_train = train_df
X_test = test_df.loc[indices_real,:]
X_test['target'] = np.zeros(X_test.shape[0])
 X_fake = test_df.loc[indices_fake,:]
X_fake['target'] = np.zeros(X_test.shape[0])
 train_length = X_train.shape[0]
target_test = X_test['target']
target_fake = X_fake['target']
if use_experimental:
np.random.seed(42)
 indices = np.arange(train_length)
 train_length = 150000
np.random.shuffle(indices)
 indices_train = indices[:train_length]
indices_test = indices[train_length:]
X_test = X_train.iloc[indices_test,:]
X_fake = X_train.iloc[indices_test,:]
target_fake = X_fake['target']
X_train = X_train.iloc[indices_train,:]
target_train = X_train['target']
target_test = X_test['target']
X_all = pd.concat([X_train, X_test])
 print(X_all.shape )<define_search_space>
model = DeepAverageNetwork(embedding_matrix=vectors, n_classes=n_classes )
sigma_fac = 0.001
sigma_base = 4
eps = 0.00000001
def get_count(X_all, X_fake):
features_count = np.zeros(( X_all.shape[0], len(features)))
 features_density = np.zeros(( X_all.shape[0], len(features)))
 features_deviation = np.zeros(( X_all.shape[0], len(features)))
 features_count_fake = np.zeros(( X_fake.shape[0], len(features)))
 features_density_fake = np.zeros(( X_fake.shape[0], len(features)))
 features_deviation_fake = np.zeros(( X_fake.shape[0], len(features)))
 sigmas = []
for i,var in enumerate(tqdm(features)) :
X_all_var_int =(X_all[var].values * 10000 ).round().astype(int)
 X_fake_var_int =(X_fake[var].values * 10000 ).round().astype(int)
 lo = X_all_var_int.min()
X_all_var_int -= lo
X_fake_var_int -= lo
hi = X_all_var_int.max() +1
counts_all = np.bincount(X_all_var_int, minlength=hi ).astype(float)
 zeros =(counts_all == 0 ).astype(int)
 before_zeros = np.concatenate([zeros[1:],[0]])
 indices_all = np.arange(counts_all.shape[0])
 sigma_scaled = counts_all.shape[0]*sigma_fac
sigma = np.power(sigma_base * sigma_base * sigma_scaled, 1/3)
 sigmas.append(sigma)
 counts_all_smooth = scipy.ndimage.filters.gaussian_filter1d(counts_all, sigma)
 deviation = counts_all /(counts_all_smooth+eps)
 indices = X_all_var_int
features_count[:,i] = counts_all[indices]
features_density[:,i] = counts_all_smooth[indices]
features_deviation[:,i] = deviation[indices]
indices_fake = X_fake_var_int
features_count_fake[:,i] = counts_all[indices_fake]
features_density_fake[:,i] = counts_all_smooth[indices_fake]
features_deviation_fake[:,i] = deviation[indices_fake]
features_count_names = [var+'_count' for var in features]
features_density_names = [var+'_density' for var in features]
features_deviation_names = [var+'_deviation' for var in features]
X_all_count = pd.DataFrame(columns=features_count_names, data = features_count)
 X_all_count.index = X_all.index
X_all_density = pd.DataFrame(columns=features_density_names, data = features_density)
 X_all_density.index = X_all.index
X_all_deviation = pd.DataFrame(columns=features_deviation_names, data = features_deviation)
 X_all_deviation.index = X_all.index
X_all = pd.concat([X_all,X_all_count, X_all_density, X_all_deviation], axis=1)
 X_fake_count = pd.DataFrame(columns=features_count_names, data = features_count_fake)
 X_fake_count.index = X_fake.index
X_fake_density = pd.DataFrame(columns=features_density_names, data = features_density_fake)
 X_fake_density.index = X_fake.index
X_fake_deviation = pd.DataFrame(columns=features_deviation_names, data = features_deviation_fake)
 X_fake_deviation.index = X_fake.index
X_fake = pd.concat([X_fake,X_fake_count, X_fake_density, X_fake_deviation], axis=1)
 features_count = features_count_names
features_density = features_density_names
features_deviation = features_deviation_names
return X_all, features_count, features_density, features_deviation, X_fake
X_all, features_count, features_density, features_deviation, X_fake = get_count(X_all, X_fake)
 print(X_all.shape )<categorify>
with torch.no_grad() :
pred = model(x )
<normalization>
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu' )
features_to_scale = [features, features_count]
def get_standardized(X_all, X_fake):
scaler = StandardScaler()
features_to_scale_flatten = [var for sublist in features_to_scale for var in sublist]
scaler.fit(X_all[features_to_scale_flatten])
 features_scaled = scaler.transform(X_all[features_to_scale_flatten])
 features_scaled_fake = scaler.transform(X_fake[features_to_scale_flatten])
 X_all[features_to_scale_flatten] = features_scaled
X_fake[features_to_scale_flatten] = features_scaled_fake
return X_all, X_fake
X_all, X_fake = get_standardized(X_all, X_fake)
 print(X_all.shape )<define_variables>
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(params=model.parameters())
 model = model.to(device)
 criterion = criterion.to(device )
<define_variables>
epochs = 10
losses = []
best_test_loss = 10.
 test_f1 = []
for n_epoch in range(epochs):
train_losses = []
test_losses = []
test_targets = []
test_pred_class = []
progress_bar = tqdm(total=len(train_loader.dataset), desc='Epoch {}'.format(n_epoch + 1))
model.train()
for x, y in train_loader:
x = x.to(device)
 y = y.to(device)
 optimizer.zero_grad()
pred = model(x)
 loss = criterion(pred, y)
 loss.backward()
optimizer.step()
train_losses.append(loss.item())
 losses.append(loss.item())
 progress_bar.set_postfix(train_loss = np.mean(losses[-500:]))
progress_bar.update(x.shape[0])
 progress_bar.close()
model.eval()
for x, y in validation_loader:
x = x.to(device)
 y = y.to(device)
 with torch.no_grad() :
pred = model(x)
 pred = pred.cpu()
y = y.cpu()
test_targets.append(y.numpy())
 test_pred_class.append(np.argmax(pred, axis=1))
loss = criterion(pred, y)
 test_losses.append(loss.item())
 mean_test_loss = np.mean(test_losses)
 test_targets = np.concatenate(test_targets ).squeeze()
test_pred_class = np.concatenate(test_pred_class ).squeeze()
f1 = f1_score(test_targets, test_pred_class, average='micro')
 test_f1.append(f1)
 print()
print('Losses: train - {:.3f}, test - {:.3f}'.format(np.mean(train_losses), mean_test_loss))
print('F1 test - {:.3f}'.format(f1))
if mean_test_loss < best_test_loss:
best_test_loss = mean_test_loss
else:
print('Early stopping')
 break
features_used = [features, features_count]<init_hyperparams>
model.eval()
predictions = []
for x, _ in test_loader:
x = x.to(device)
 with torch.no_grad() :
pred = model(x)
 pred = pred.cpu()
predictions.append(np.argmax(pred, axis=1))
predictions = np.concatenate(predictions ).squeeze()
params = {
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 1,
'learning_rate': 0.08,
'max_depth': -1,
'metric':'binary_logloss',
'num_leaves': 4,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'reg_alpha': 2,
'reg_lambda': 0,
'verbosity': 1,
'max_bin':256,
}
reg_alpha_values = [0.75, 1, 2, 3]
reg_alpha_var = [3, 0, 2, 3, 2, 0, 1, 1, 3, 2, 2, 0, 2, 0, 2, 2, 2, 1, 1, 2, 1, 2, 3, 3, 2, 1, 3, 1, 3, 2, 2, 3, 1, 1, 3, 2, 0, 1, 0, 2, 1, 1, 2, 3, 0, 3, 3, 3, 2, 0, 3, 1, 3, 1, 1, 0, 2, 2, 0, 0, 0, 1, 2, 1, 0, 1, 3, 2, 0, 2, 1, 2, 0, 0, 1, 3, 3, 1, 2, 3, 3, 2, 0, 1, 2, 3, 3, 2, 3, 3, 0, 0, 3, 0, 1, 0, 1, 0, 2, 3, 1, 0, 3, 1, 3, 2, 3, 1, 3, 3, 3, 1, 3, 2, 3, 2, 1, 0, 1, 2, 0, 3, 0, 3, 0, 3, 2, 1, 0, 0, 2, 2, 2, 0, 1, 0, 0, 2, 3, 2, 2, 1, 1, 0, 1, 2, 2, 2, 1, 0, 2, 3, 2, 3, 1, 1, 3, 1, 1, 2, 1, 2, 0, 3, 1, 3, 3, 2, 0, 1, 3, 3, 0, 1, 0, 3, 1, 3, 1, 3, 0, 3, 0, 3, 1, 0, 0, 0, 3, 0, 3, 0, 0, 2, 0, 3, 1, 0, 3, 2]
max_bin_values = [256, 512, 1024]
max_bin_var = [0, 0, 1, 0, 0, 0, 2, 0, 0, 2, 0, 2, 0, 0, 1, 1, 1, 0, 0, 0, 0, 2, 2, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 2, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 2, 1, 1, 1, 1, 0, 0, 0, 0, 1, 2, 1, 0, 0, 1, 2, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 2, 0, 1, 1, 0, 1, 0, 0, 1, 2, 1, 2, 1, 0, 0, 1, 0, 2, 0, 1, 0, 0, 2, 1, 1, 1, 0, 0, 0, 2, 0, 0, 2, 1, 0, 0, 1, 0, 1, 2, 0, 0, 0, 0, 0, 2, 2, 2, 2, 1, 1, 2, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 2, 0, 1, 0, 1, 1, 0, 2, 1, 1, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 1, 0, 1, 2, 0, 0, 0, 0, 2, 0, 0, 2, 0, 1, 1, 0, 2, 0, 0, 0, 1, 2, 0, 0, 1, 0, 2]
learning_rate_values = [0.06, 0.08, 0.12]
learning_rate_var = [2, 2, 2, 1, 2, 2, 2, 0, 1, 2, 0, 2, 2, 2, 0, 2, 2, 0, 2, 1, 2, 2, 2, 2, 2, 0, 1, 0, 2, 0, 0, 2, 0, 2, 2, 2, 1, 2, 0, 0, 2, 0, 0, 1, 2, 1, 2, 0, 0, 2, 1, 2, 2, 2, 2, 0, 0, 2, 1, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 0, 2, 2, 2, 0, 2, 2, 1, 0, 1, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 1, 1, 1, 2, 0, 2, 0, 2, 0, 2, 1, 0, 0, 1, 2, 0, 2, 2, 2, 0, 2, 2, 2, 2, 1, 0, 2, 1, 2, 2, 1, 2, 0, 2, 0, 2, 2, 2, 2, 2, 2, 1, 2, 1, 0, 2, 1, 1, 2, 2, 2, 2, 0, 2, 1, 2, 1, 2, 2, 2, 2, 2, 2, 2, 1, 1, 0, 1, 2, 0, 2, 2, 0, 1, 2, 2, 2, 1, 0, 1, 2, 1, 2, 1, 1, 1, 2, 1, 2, 1, 0, 0, 2, 0, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1]
num_leaves_values = [3, 4, 5]
num_leaves_var = [1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 2, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 2, 1, 2, 0, 0, 0, 0, 0, 1, 1, 0, 0, 2, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 2, 0, 0, 0, 0, 1, 0, 1, 2, 1, 1, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 2, 0, 1, 0, 2, 2, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 2, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 1, 2, 1, 1, 0, 0, 0, 2, 1, 2] 
<define_variables>
test['main_category'] = predictions
n_folds = 5
early_stopping_rounds=10
settings = [4]
np.random.seed(47)
 settings_best_ind = []
def train_trees() :
preds_oof = np.zeros(( len(X_train), len(features)))
 preds_test = np.zeros(( len(X_test), len(features)))
 preds_train = np.zeros(( len(X_train), len(features)))
 preds_fake = np.zeros(( len(X_fake), len(features)))
 features_used_flatten = [var for sublist in features_used for var in sublist]
X_train_used = X_train[features_used_flatten]
X_test_used = X_test[features_used_flatten]
X_fake_used = X_fake[features_used_flatten]
for i in range(len(features)) :
params['max_bin'] = max_bin_values[max_bin_var[i]]
params['learning_rate'] = learning_rate_values[learning_rate_var[i]]
params['reg_alpha'] = reg_alpha_values[reg_alpha_var[i]]
params['num_leaves'] = num_leaves_values[num_leaves_var[i]]
features_train = [feature_set[i] for feature_set in features_used]
print(f'Training on: {features_train}')
 folds = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=np.random.randint(100000))
list_folds = list(folds.split(X_train_used.values, target_train.values))
preds_oof_temp = np.zeros(( preds_oof.shape[0], len(settings)))
 preds_test_temp = np.zeros(( preds_test.shape[0], len(settings)))
 preds_train_temp = np.zeros(( preds_train.shape[0], len(settings)))
 preds_fake_temp = np.zeros(( preds_fake.shape[0], len(settings)))
 scores = []
for j, setting in enumerate(settings):
print('
setting: ', setting)
 for k,(trn_idx, val_idx)in enumerate(list_folds):
print("Fold: {}".format(k+1), end="")
 trn_data = lgb.Dataset(X_train_used.iloc[trn_idx][features_train], label=target_train.iloc[trn_idx])
 val_data = lgb.Dataset(X_train_used.iloc[val_idx][features_train], label=target_train.iloc[val_idx])
 clf = lgb.train(params, trn_data, 2000, valid_sets=[trn_data, val_data], verbose_eval=False, early_stopping_rounds=early_stopping_rounds)
 prediction_val1 = clf.predict(X_train_used.iloc[val_idx][features_train])
 prediction_test1 = clf.predict(X_test_used[features_train])
 prediction_train1 = clf.predict(X_train_used.iloc[trn_idx][features_train])
 prediction_fake1 = clf.predict(X_fake_used[features_train])
 s1 = roc_auc_score(target_train.iloc[val_idx], prediction_val1)
 s1_log = log_loss(target_train.iloc[val_idx], prediction_val1)
 print(' - val AUC: {:<8.4f} - loss: {:<8.3f}'.format(s1, s1_log*1000), end='')
 if use_experimental:
s1_test = roc_auc_score(target_test, prediction_test1)
 s1_log_test = log_loss(target_test, prediction_test1)
 print(' - test AUC: {:<8.4f} - loss: {:<8.3f}'.format(s1_test, s1_log_test*1000), end='')
 s1_train = roc_auc_score(target_train.iloc[trn_idx], prediction_train1)
 s1_log_train = log_loss(target_train.iloc[trn_idx], prediction_train1)
 print(' - train AUC: {:<8.4f} - loss: {:<8.3f}'.format(s1_train, s1_log_train*1000), end='')
 if use_experimental:
print('',clf.feature_importance() , end='')
 print('')
 preds_oof_temp[val_idx,j] += np.sqrt(prediction_val1 - prediction_val1.mean() + 0.1)
 preds_test_temp[:,j] += np.sqrt(prediction_test1 - prediction_test1.mean() + 0.1)/ n_folds
preds_train_temp[trn_idx,j] += np.sqrt(prediction_train1 - prediction_train1.mean() + 0.1)/(n_folds-1)
 preds_fake_temp[:,j] += np.sqrt(prediction_fake1 - prediction_fake1.mean() + 0.1)/ n_folds
score_setting = roc_auc_score(target_train, preds_oof_temp[:,j])
 score_setting_log = 1000*log_loss(target_train, np.exp(preds_oof_temp[:,j]))
scores.append(score_setting_log)
 print("Score: - val AUC: {:<8.4f} - loss: {:<8.3f}".format(score_setting, score_setting_log), end='')
 if use_experimental:
score_setting_test = roc_auc_score(target_test, preds_test_temp[:,j])
 score_setting_log_test = 1000*log_loss(target_test, np.exp(preds_test_temp[:,j]))
print(" - test AUC: {:<8.4f} - loss: {:<8.3f}".format(score_setting_test, score_setting_log_test), end='')
 score_setting_train = roc_auc_score(target_train, preds_train_temp[:,j])
 score_setting_log_train = 1000*log_loss(target_train, np.exp(preds_train_temp[:,j]))
print(" - train AUC: {:<8.4f} - loss: {:<8.3f}".format(score_setting_train, score_setting_log_train))
best_ind = np.argmin(scores)
 settings_best_ind.append(best_ind)
 preds_oof[:,i] = preds_oof_temp[:,best_ind]
preds_test[:,i] = preds_test_temp[:,best_ind]
preds_train[:,i] = preds_train_temp[:,best_ind]
preds_fake[:,i] = preds_fake_temp[:,best_ind]
print('
best setting: ', settings[best_ind])
 preds_oof_cum = preds_oof[:,:i+1].mean(axis=1)
 print("Cum CV val : {:<8.4f} - loss: {:<8.3f}".format(roc_auc_score(target_train, preds_oof_cum), 1000*log_loss(target_train, np.exp(preds_oof_cum))))
if use_experimental:
preds_test_cum = preds_test[:,:i+1].mean(axis=1)
 print("Cum CV test : {:<8.4f} - loss: {:<8.3f}".format(roc_auc_score(target_test, preds_test_cum), 1000*log_loss(target_test, np.exp(preds_test_cum))))
preds_train_cum = preds_train[:,:i+1].mean(axis=1)
 print("Cum CV train: {:<8.4f} - loss: {:<8.3f}".format(roc_auc_score(target_train, preds_train_cum), 1000*log_loss(target_train, np.exp(preds_train_cum))))
print('*****' * 10 + '
')
 return preds_oof, preds_test, preds_train, preds_fake
preds_oof, preds_test, preds_train, preds_fake = train_trees()<compute_test_metric>
test = test[['index', 'main_category']]
<define_variables><EOS> 
test.to_csv('submission.csv', index=False )