Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<feature_engineering>
sys.path.append('.. /input/iterative-stratification/iterative-stratification-master')
 warnings.filterwarnings('ignore' )
class Iter_Valid(object):
def __init__(self, df, max_user=1000):
df = df.reset_index(drop=True)
 self.df = df
self.user_answer = df['user_answer'].astype(str ).values
self.answered_correctly = df['answered_correctly'].astype(str ).values
df['prior_group_responses'] = "[]"
df['prior_group_answers_correct'] = "[]"
self.sample_df = df[df['content_type_id'] == 0][['row_id']]
self.sample_df['answered_correctly'] = 0
self.len = len(df)
 self.user_id = df.user_id.values
self.task_container_id = df.task_container_id.values
self.content_type_id = df.content_type_id.values
self.max_user = max_user
self.current = 0
self.pre_user_answer_list = []
self.pre_answered_correctly_list = []
def __iter__(self):
return self
def fix_df(self, user_answer_list, answered_correctly_list, pre_start):
df= self.df[pre_start:self.current].copy()
sample_df = self.sample_df[pre_start:self.current].copy()
df.loc[pre_start,'prior_group_responses'] = '[' + ",".join(self.pre_user_answer_list)+ ']'
df.loc[pre_start,'prior_group_answers_correct'] = '[' + ",".join(self.pre_answered_correctly_list)+ ']'
self.pre_user_answer_list = user_answer_list
self.pre_answered_correctly_list = answered_correctly_list
return df, sample_df
def __next__(self):
added_user = set()
pre_start = self.current
pre_added_user = -1
pre_task_container_id = -1
pre_content_type_id = -1
user_answer_list = []
answered_correctly_list = []
while self.current < self.len:
crr_user_id = self.user_id[self.current]
crr_task_container_id = self.task_container_id[self.current]
crr_content_type_id = self.content_type_id[self.current]
if crr_user_id in added_user and(crr_user_id != pre_added_user or(crr_task_container_id != pre_task_container_id and crr_content_type_id == 0 and pre_content_type_id == 0)) :
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 if len(added_user)== self.max_user:
if crr_user_id == pre_added_user and(crr_task_container_id == pre_task_container_id or crr_content_type_id == 1):
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
continue
else:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 added_user.add(crr_user_id)
 pre_added_user = crr_user_id
pre_task_container_id = crr_task_container_id
pre_content_type_id = crr_content_type_id
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
if pre_start < self.current:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 else:
raise StopIteration()<predict_on_test>
print("torch version:", torch.__version__ )
validaten_flg = False
if validaten_flg:
target_df = pd.read_pickle('.. /input/riiid-cross-validation-files/cv1_train.pickle')
 target_df = target_df[50_000_000:52_500_000]
iter_test = Iter_Valid(target_df,max_user=1000)
 predicted = []
def set_predict(df):
predicted.append(df)
 else:
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict<load_pretrained>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
lgb_model = pickle.load(open('/kaggle/input/riid-preprocessed/lgb_model.pkl', 'rb'))<load_pretrained>
ROOT = ".. /input/lish-moa"
prior_b_dict = pickle_load_dill('.. /input/riid-preprocessed/prior_b_dict.pickle')
 diagnostic_u_dict = pickle_load_dill('.. /input/riid-preprocessed/diagnostic_u_dict.pickle')
 question_trueskill_dict = pickle_load_dill('.. /input/riid-preprocessed/question_trueskill_dict.pickle')
 user_trueskill_dict = pickle_load_dill('.. /input/riid-preprocessed/user_trueskill_dict.pickle' )<data_type_conversions>
df_train_features = pd.read_csv(f'{ROOT}/train_features.csv')
 df_train_targets_scored = pd.read_csv(f'{ROOT}/train_targets_scored.csv')
 df_train_targets_nonscored = pd.read_csv(f'{ROOT}/train_targets_nonscored.csv')
 df_train_drug = pd.read_csv(f'{ROOT}/train_drug.csv')
 df_test_features = pd.read_csv(f'{ROOT}/test_features.csv')
 df_sample_submission = pd.read_csv(f'{ROOT}/sample_submission.csv' )
pbar = tqdm(total=2500000)
 previous_test_df = None
counter = 0
conn = sqlite3.connect(dbname)
 for(test_df, sample_prediction_df)in iter_test:
counter += 1
if previous_test_df is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_feats_sqlite(previous_test_df, conn)
 update_trueskill(previous_test_df, user_trueskill_dict, question_trueskill_dict, verbose=False)
 if not counter % 100:
conn.commit()
test_df.timestamp=(test_df.timestamp/1000 ).astype('int32')
 test_df = add_prior_b(test_df, questions_cb, questions_bpt, prior_b_dict,verbose=False)
 test_df['col_for_merge']=(test_df['content_id'].astype('int32')*10+test_df['content_type_id'] ).astype('int32')
 test_df = test_df.merge(content_df, how = 'left', on = 'col_for_merge')
 test_df.content_id=test_df.content_id.astype('int32')
 test_df['prior_question_elapsed_time'] = test_df['prior_question_elapsed_time'].fillna(25439.41)
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(False ).astype('int8')
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = add_feats_without_update_sqlite(test_df, conn)
 test_df = add_is_diagnostic(test_df, diagnostic_u_dict,verbose=False)
 test_df = add_trueskill_without_update(test_df, user_trueskill_dict, question_trueskill_dict, verbose=False)
 test_df = add_new_columns(test_df)
 test_df = fillna_df(test_df)
 lgb_result = lgb_model.predict(test_df[FEATS])
 test_df[TARGET] = lgb_result
set_predict(test_df[['row_id', TARGET]])
 pbar.update(len(test_df))<load_from_csv>
col_genes = [col for col in df_train_features.columns.to_list() if col[:2] == "g-"]
col_cells = [col for col in df_train_features.columns.to_list() if col[:2] == "c-"]
col_cats = ["cp_time", "cp_dose"]
col_targets = df_train_targets_scored.columns.to_list() [1:]
print("-"*25)
 print("Number of Gene columns: ", len(col_genes))
print("Number of Cell columns: ", len(col_cells))
print("Number of Categorical columns:", len(col_cats))
print("-"*25)
 print("Number of Target columns: ", len(col_targets))
print("-"*25 )
data_types_dict = {
'user_id': 'int32',
'content_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'}
target = 'answered_correctly'
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()
train_df = train_df[train_df[target] != -1].reset_index(drop=True)
 train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict )<feature_engineering>
print("Data explore:")
 print("-"*25)
 print("Number of unique in sig_id: %5d" % df_train_features.sig_id.unique().shape[0], "| Examples:", df_train_features.sig_id.unique() [:3])
 print("Number of unique in cp_type: %5d" % df_train_features.cp_type.unique().shape[0], "| Examples:", df_train_features.cp_type.unique() [:3])
 print("Number of unique in cp_time: %5d" % df_train_features.cp_time.unique().shape[0], "| Examples:", df_train_features.cp_time.unique() [:3])
 print("Number of unique in cp_dose: %5d" % df_train_features.cp_dose.unique().shape[0], "| Examples:", df_train_features.cp_dose.unique() [:3])
 print("-"*25)
 print("Most frequent targets:")
 print("-"*25)
 df_train_targets_scored.sum() [1:].sort_values(ascending=False )
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df.drop(columns=['lag'], inplace=True)
 user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])
 content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count'])
 train_df = train_df.groupby('user_id' ).tail(24 ).reset_index(drop=True)
 questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16', 'part': 'int8'} 
)
 train_df = pd.merge(train_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 train_df.drop(columns=['question_id'], inplace=True)
 train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_id'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 valid_df = train_df.groupby('user_id' ).tail(6)
 train_df.drop(valid_df.index, inplace=True )<train_model>
df_train = df_train_features.merge(df_train_targets_scored, how="left", on="sig_id")
 df_train = df_train.merge(df_train_drug, on="sig_id", how="left")
 df_train = df_train.dropna()
df_test = df_test_features.copy()
print(df_train.shape)
 print(df_test.shape)
 df_train.head(2 )
features = [
'content_id',
'prior_question_elapsed_time',
'prior_question_had_explanation',
'user_correctness',
'part',
'content_count'
]
params = {
'objective': 'binary',
'seed': 42,
'metric': 'auc',
'learning_rate': 0.05,
'max_bin': 800,
'num_leaves': 80
}
tr_data = lgb.Dataset(train_df[features], label=train_df[target])
 va_data = lgb.Dataset(valid_df[features], label=valid_df[target])
 model = lgb.train(
 params,
tr_data,
num_boost_round=10000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
verbose_eval=50 
)
 lgb.plot_importance(model, importance_type='gain')
 plt.show()<feature_engineering>
NFOLDS = 5
SEED = 42
if "fold" in df_train.columns: del df_train['fold']
vc = df_train.drug_id.value_counts()
vc1 = vc.loc[vc<=18].index.sort_values()
vc2 = vc.loc[vc>18].index.sort_values()
dct1 = {}; dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=SEED)
 tmp = df_train.groupby('drug_id')[col_targets].mean().loc[vc1]
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[col_targets])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=SEED)
 tmp = df_train.loc[df_train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[col_targets])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 df_train['fold'] = df_train.drug_id.map(dct1)
 df_train.loc[df_train.fold.isna() ,'fold'] =\
df_train.loc[df_train.fold.isna() ,'sig_id'].map(dct2)
 df_train.fold = df_train.fold.astype('int8' )
for(test_df, sample_prediction_df)in iter_test:
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
targets = prior_test_df[target].values
for user_id, content_id, answered_correctly in zip(user_ids, content_ids, targets):
if user_id in user_agg.index:
user_agg.loc[user_id, 'sum'] += answered_correctly
user_agg.loc[user_id, 'count'] += 1
else:
user_agg.loc[user_id] = [answered_correctly, 1]
if content_id in content_agg.index:
content_agg.loc[content_id, 'sum'] += answered_correctly
content_agg.loc[content_id, 'count'] += 1
else:
content_agg.loc[content_id] = [answered_correctly, 1]
prior_test_df = test_df.copy()
test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(False ).astype('bool')
 test_df['user_correctness'] = test_df['user_id'].map(user_agg['sum'] / user_agg['count'])
 test_df['content_count'] = test_df['content_id'].map(content_agg['count'] ).fillna(1)
 test_df['content_id'] = test_df['content_id'].map(content_agg['sum'] / content_agg['count'] ).fillna(0.7)
 test_df['answered_correctly'] = model.predict(test_df[features])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])
<set_options>
df_train.loc[:, 'cp_dose'] = df_train.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 df_test.loc[:, 'cp_dose'] = df_test.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 col_input = col_cats + col_cells + col_genes
pd.set_option('display.max_columns', 50)
<define_variables>
class MoADataset:
def __init__(self, features_cat, features_gene, features_cell, targets):
self.features_cat = features_cat
self.features_gene = features_gene
self.features_cell = features_cell
self.targets = targets
def __len__(self):
return(self.features_cat.shape[0])
 def __getitem__(self, idx):
dct = {
'x1' : torch.tensor(self.features_cat[idx, :], dtype=torch.float),
'x2' : torch.tensor(self.features_gene[idx, :], dtype=torch.float),
'x3' : torch.tensor(self.features_cell[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features_cat, features_gene, features_cell):
self.features_cat = features_cat
self.features_gene = features_gene
self.features_cell = features_cell
def __len__(self):
return(self.features_cat.shape[0])
 def __getitem__(self, idx):
dct = {
'x1' : torch.tensor(self.features_cat[idx, :], dtype=torch.float),
'x2' : torch.tensor(self.features_gene[idx, :], dtype=torch.float),
'x3' : torch.tensor(self.features_cell[idx, :], dtype=torch.float),
}
return dct
features = [
'content_id',
'prior_question_elapsed_time',
'prior_question_had_explanation',
'user_correctness',
'content_count',
'part',
'content_mean',
'cumcount_u',
'cumcount_p',
'attempt',
'part_avg',
'timestamp_diff1',
'timestamp_diff2',
'cluster_id',
'cumcount_cl',
'target_lag',
'cluster0_avg',
'cluster1_avg',
'cluster2_avg',
'prior_tag',
'task_num',
'user_rating',
'time_mean_diff',
]
target = 'answered_correctly'
features_dtypes = {
'content_id': 'int16',
'content_mean': 'float32',
'prior_question_elapsed_time': 'float64',
'prior_question_had_explanation': 'bool',
'user_correctness': 'float32',
'content_count': 'int32',
'part': 'int8',
'cumcount_u': 'uint16',
'cumcount_p': 'uint16',
'attempt': 'uint16',
'part_avg': 'float32',
'timestamp_diff1': 'float64',
'timestamp_diff2': 'float64',
'cluster_id': 'int8',
'cumcount_cl': 'uint16',
'target_lag': 'int8',
'cluster0_avg': 'float32',
'cluster1_avg': 'float32',
'cluster2_avg': 'float32',
'prior_tag': 'int16',
'task_num': 'int8',
'user_rating': 'float32',
'time_mean_diff': 'float32',
}<load_pretrained>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
input1, input2, input3, targets = data['x1'].to(device), data['x2'].to(device), data['x3'].to(device), data['y'].to(device)
 outputs = model(input1, input2, input3)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
if scheduler: scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
input1, input2, input3, targets = data['x1'].to(device), data['x2'].to(device), data['x3'].to(device), data['y'].to(device)
 outputs = model(input1, input2, input3)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
input1, input2, input3 = data['x1'].to(device), data['x2'].to(device), data['x3'].to(device)
 with torch.no_grad() :
outputs = model(input1, input2, input3)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
time_dict1 = joblib.load(".. /input/riiid-inf-data/time_dict1.pkl.zip")
 time_dict2 = joblib.load(".. /input/riiid-inf-data/time_dict2.pkl.zip")
 time_dict3 = joblib.load(".. /input/riiid-inf-data/time_dict3.pkl.zip")
 questions_df = pd.read_pickle('.. /input/riiid-inf-data/questions_df.pickle')
 lectures_df = pd.read_pickle('.. /input/riiid-inf-data/lectures_df.pickle')
 part_null_data = pd.read_pickle('.. /input/riiid-inf-data/part_null_data.pickle')
 cluster_null_data = pd.read_pickle('.. /input/riiid-inf-data/cluster_null_data.pickle' )<load_pretrained>
def logloss(y_true, y_pred, eps=1e-15):
y_pred = np.clip(y_pred, eps, 1 - eps)
 return -(y_true * np.log(y_pred)+(1-y_true)* np.log(1-y_pred)).mean()
user_dict_sum = joblib.load(".. /input/riiid-dict-data/user_dict_sum.pkl.zip")
 user_dict_count = joblib.load(".. /input/riiid-dict-data/user_dict_count.pkl.zip")
 part_dict_sum = joblib.load(".. /input/riiid-dict-data/part_dict_sum.pkl.zip")
 part_dict_count = joblib.load(".. /input/riiid-dict-data/part_dict_count.pkl.zip")
 cluster_dict_sum = joblib.load(".. /input/riiid-dict-data/cluster_dict_sum.pkl.zip")
 cluster_dict_count = joblib.load(".. /input/riiid-dict-data/cluster_dict_count.pkl.zip")
 lag_dict = joblib.load(".. /input/riiid-dict-data/lag_dict.pkl.zip")
 last_lecture_dict = joblib.load(".. /input/riiid-dict-data/last_lecture_dict.pkl.zip")
 content_mean_sum_dict = joblib.load(".. /input/riiid-dict-data/content_mean_sum_dict.pkl.zip")
 time_adm_dict = joblib.load(".. /input/riiid-dict-data/time_adm_dict.pkl.zip" )<categorify>
class Model(nn.Module):
def __init__(self, num_features_cat, num_features_gene, num_features_cell, num_targets, hidden_size, dropout_rate=0.5, activation=F.relu):
super(Model, self ).__init__()
def get_sequential_layers(num_features, n_layers, hidden_size, dropout_rate):
layer_list = []
in_ = num_features
for i in range(n_layers):
layer_list.append(
 torch.nn.Sequential 
(
 torch.nn.BatchNorm1d(in_),
torch.nn.Dropout(dropout_rate),
torch.nn.Linear(in_, hidden_size), 
)
)
 in_ = hidden_size
return layer_list
self.layer_list_cat = torch.nn.ModuleList(get_sequential_layers(num_features_cat , 2, hidden_size, dropout_rate))
self.layer_list_gene = torch.nn.ModuleList(get_sequential_layers(num_features_gene, 4, hidden_size, dropout_rate))
self.layer_list_cell = torch.nn.ModuleList(get_sequential_layers(num_features_cell, 3, hidden_size, dropout_rate))
self.activation = activation
self.batch_norm_out = nn.BatchNorm1d(hidden_size*3)
 self.dropout_out = nn.Dropout(dropout_rate)
 self.dense_out = nn.Linear(hidden_size*3, num_targets)
 def forward(self, x1, x2, x3):
for batchnorm, dropout, linear in self.layer_list_cat:
x1 = batchnorm(x1)
 x1 = dropout(x1)
 x1 = linear(x1)
 x1 = self.activation(x1)
 for batchnorm, dropout, linear in self.layer_list_gene:
x2 = batchnorm(x2)
 x2 = dropout(x2)
 x2 = linear(x2)
 x2 = self.activation(x2)
 for batchnorm, dropout, linear in self.layer_list_cell:
x3 = batchnorm(x3)
 x3 = dropout(x3)
 x3 = linear(x3)
 x3 = self.activation(x3)
 x = torch.cat([x1, x2, x3], dim=-1)
 x = self.batch_norm_out(x)
 x = self.dropout_out(x)
 x = self.dense_out(x)
 return x
def get_state() :
data = pd.read_pickle('.. /input/riiid-inf-data/state_data.pickle')
 state = dict()
for user_id in data['user_id'].unique() :
state[user_id] = {}
user_content = data.groupby('user_id')['content_id'].apply(np.array ).apply(np.sort ).apply(np.unique)
 user_attempts = data.groupby(['user_id', 'content_id'])['content_id'].count().astype(np.uint8 ).groupby('user_id' ).apply(np.array ).values
user_attempts -= 1
del data
gc.collect()
for user_id, content, attempt in zip(state.keys() , user_content, user_attempts):
state[user_id]['user_content_attempts'] = dict(zip(content, attempt))
del user_content, user_attempts
gc.collect()
return state
state = get_state()<feature_engineering>
def run_training(fold, seed, PARAMS, df_train, DEVICE):
seed_everything(seed)
 train_ = df_train.copy()
trn_idx = train_[train_['fold'] != fold].index
val_idx = train_[train_['fold'] == fold].index
train_df = train_[train_['fold'] != fold].reset_index(drop=True)
 valid_df = train_[train_['fold'] == fold].reset_index(drop=True)
 x_train_cat, x_train_gene, x_train_cell, y_train = train_df[col_cats].values, train_df[col_genes].values, train_df[col_cells].values, train_df[col_targets].values
x_valid_cat, x_valid_gene, x_valid_cell, y_valid = valid_df[col_cats].values, valid_df[col_genes].values, valid_df[col_cells].values, valid_df[col_targets].values
train_dataset = MoADataset(x_train_cat, x_train_gene, x_train_cell, y_train)
 valid_dataset = MoADataset(x_valid_cat, x_valid_gene, x_valid_cell, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=PARAMS["BATCH_SIZE"], shuffle=True, drop_last=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=PARAMS["BATCH_SIZE"], shuffle=False)
 model = Model(
 num_features_cat=len(col_cats),
num_features_gene=len(col_genes),
num_features_cell=len(col_cells),
num_targets=len(col_targets),
hidden_size=PARAMS["hidden_size"],
dropout_rate=PARAMS["dropout_rate"],
activation=PARAMS["activation"], 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=PARAMS["LEARNING_RATE"], weight_decay=PARAMS["WEIGHT_DECAY"])
 scheduler = None
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 if VERBOSE and epoch % VERBOSE == 0:
print(f"FOLD: %d, EPOCH: %3d | train_loss: %.5f | valid_loss: %.5f"%(fold,epoch,train_loss,valid_loss))
if valid_loss < best_loss:
if epoch > 1:
os.rename(f"SEED_{seed}_FOLD{fold}_0.pth", f"SEED_{seed}_FOLD{fold}_1.pth")
 best_loss = valid_loss
torch.save(model.state_dict() , f"SEED_{seed}_FOLD{fold}_0.pth")
 early_step = 0
elif(EARLY_STOP == True):
early_step += 1
if(early_step >= EARLY_STOPPING_STEPS):
break
x_test_cat, x_test_gene, x_test_cell = df_test[col_cats].values, df_test[col_genes].values, df_test[col_cells].values
testdataset = TestDataset(x_test_cat, x_test_gene, x_test_cell)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=PARAMS["BATCH_SIZE"], shuffle=False)
 oof = np.zeros(( df_train.shape[0], len(col_targets)))
 predictions = np.zeros(( df_test.shape[0], len(col_targets)))
 for nb in range(NBEST):
model = Model(
 num_features_cat=len(col_cats),
num_features_gene=len(col_genes),
num_features_cell=len(col_cells),
num_targets=len(col_targets),
hidden_size=PARAMS["hidden_size"],
dropout_rate=PARAMS["dropout_rate"],
activation=PARAMS["activation"], 
)
 model_path = f"SEED_{seed}_FOLD{fold}_{nb}.pth"
if not os.path.isfile(model_path):
continue
model.load_state_dict(torch.load(model_path))
model.to(DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 oof[val_idx] += valid_preds / NBEST
predictions += inference_fn(model, testloader, DEVICE)/ NBEST
score = logloss(y_valid, oof[val_idx])
 print("FOLD: %d, Log Loss: %.5f"%(fold, score))
print("-"*15)
 return oof, predictions, score
def get_attempt(test):
attempt = []
for idx,(user_id, content_id)in enumerate(test[['user_id', 'content_id']].values):
if user_id in state:
if content_id in state[user_id]['user_content_attempts']:
state[user_id]['user_content_attempts'][content_id] = min(6, state[user_id]['user_content_attempts'][content_id] + 1)
 else:
state[user_id]['user_content_attempts'][content_id] = 0
else:
dict_keys = ['user_content_attempts']
dict_default_vals = [dict(zip([content_id],[0])) ]
state[user_id] = dict(zip(dict_keys, dict_default_vals))
attempt.append(state[user_id]['user_content_attempts'][content_id])
 return attempt<feature_engineering>
def run_k_fold(NFOLDS, seed, PARAMS, df_train, DEVICE):
oof = np.zeros(( df_train.shape[0], len(col_targets)))
 predictions = np.zeros(( df_test.shape[0], len(col_targets)))
 scores = []
for fold in range(NFOLDS):
torch.cuda.empty_cache()
oof_, pred_, score_ = run_training(fold, seed, PARAMS, df_train, DEVICE)
 predictions += pred_ / NFOLDS
oof += oof_
scores.append(score_)
 return oof, predictions, scores
def get_timestamp_diff(test):
timestamp_diff1 = []
timestamp_diff2 = []
for user_id, timestamp in test[['user_id', 'timestamp']].values:
if user_id in time_dict1:
if timestamp > time_dict1[user_id]:
if time_dict2[user_id] is np.nan:
timestamp_diff1.append(timestamp - time_dict1[user_id])
 timestamp_diff2.append(np.nan)
 time_dict3[user_id] = time_dict2[user_id]
time_dict2[user_id] = time_dict1[user_id]
time_dict1[user_id] = timestamp
else:
timestamp_diff1.append(timestamp - time_dict1[user_id])
 timestamp_diff2.append(timestamp - time_dict2[user_id])
 time_dict3[user_id] = time_dict2[user_id]
time_dict2[user_id] = time_dict1[user_id]
time_dict1[user_id] = timestamp
else:
if time_dict2[user_id] is np.nan:
timestamp_diff1.append(np.nan)
 timestamp_diff2.append(np.nan)
 elif time_dict3[user_id] is np.nan:
timestamp_diff1.append(timestamp - time_dict2[user_id])
 timestamp_diff2.append(np.nan)
 else:
timestamp_diff1.append(timestamp - time_dict2[user_id])
 timestamp_diff2.append(timestamp - time_dict3[user_id])
 else:
timestamp_diff1.append(np.nan)
 timestamp_diff2.append(np.nan)
 time_dict1[user_id] = timestamp
time_dict2[user_id] = np.nan
time_dict3[user_id] = np.nan
return timestamp_diff1, timestamp_diff2<define_variables>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 101
EARLY_STOPPING_STEPS = 30
EARLY_STOP = True
NFOLDS = 5
VERBOSE = 10
NBEST = 2
SEED = 42
PARAMS = {
'BATCH_SIZE': 256,
'LEARNING_RATE': 1e-2,
'WEIGHT_DECAY': 1e-6,
'hidden_size': 512,
'dropout_rate': 0.4,
'activation': F.relu,
}
def get_user_data(test):
user_correctness = []
cumcount_u = []
part_avg = []
cumcount_p = []
cluster0_avg = []
cluster1_avg = []
cluster2_avg = []
cumcount_cl = []
target_lag = []
user_rating = []
for user_id, part, cluster_id, content_mean in test[['user_id', 'part', 'cluster_id', 'content_mean']].values:
try:
part_null = part_null_data[part]
except:
part_null = part_null_data.mean()
try:
cluster0_null = cluster_null_data[0]
except:
cluster0_null = cluster_null_data.mean()
try:
cluster1_null = cluster_null_data[1]
except:
cluster1_null = cluster_null_data.mean()
try:
cluster2_null = cluster_null_data[2]
except:
cluster2_null = cluster_null_data.mean()
if user_id in user_dict_sum:
user_correctness.append(user_dict_sum[user_id] / user_dict_count[user_id])
 cumcount_u.append(min(7500, user_dict_count[user_id]))
user_rating.append(( user_dict_sum[user_id] - content_mean_sum_dict[user_id])/ user_dict_count[user_id])
 else:
user_correctness.append(0.68)
 cumcount_u.append(0)
 user_rating.append(0)
 k =(user_id, part)
 if k in part_dict_sum:
part_avg.append(part_dict_sum[k] / part_dict_count[k])
 cumcount_p.append(min(7500, part_dict_count[k]))
else:
part_avg.append(part_null)
 cumcount_p.append(0)
 k =(user_id, cluster_id)
 if k in cluster_dict_sum:
cumcount_cl.append(min(7500, cluster_dict_count[k]))
else:
cumcount_cl.append(0)
 k =(user_id, 0)
 if k in cluster_dict_sum:
cluster0_avg.append(cluster_dict_sum[k] / cluster_dict_count[k])
 else:
cluster0_avg.append(cluster0_null)
 k =(user_id, 1)
 if k in cluster_dict_sum:
cluster1_avg.append(cluster_dict_sum[k] / cluster_dict_count[k])
 else:
cluster1_avg.append(cluster1_null)
 k =(user_id, 2)
 if k in cluster_dict_sum:
cluster2_avg.append(cluster_dict_sum[k] / cluster_dict_count[k])
 else:
cluster2_avg.append(cluster2_null)
 if user_id in lag_dict:
target_lag.append(lag_dict[user_id])
 else:
target_lag.append(1)
 return user_correctness, cumcount_u, part_avg, cumcount_p, cluster0_avg, cluster1_avg, cluster2_avg, cumcount_cl, target_lag, user_rating<feature_engineering>
oof = np.zeros(( df_train.shape[0], len(col_targets)))
 predictions = np.zeros(( df_test.shape[0], len(col_targets)))
 st = time()
oof, predictions, scores_ = run_k_fold(NFOLDS, SEED, PARAMS, df_train, DEVICE)
 score_oof = logloss(df_train[col_targets].values, oof)
 print("-"*15)
 print("STD: %.6f | OOF: %6f | CV: %6f | done in %.1f min"%(np.std(scores_), np.mean(scores_), score_oof,(time() - st)/60))
df_test[col_targets] = predictions
<feature_engineering><EOS> 
df_test.loc[df_test.cp_type == "ctl_vehicle", col_targets] = 0
df_test[["sig_id"] + col_targets].to_csv("submission.csv", index=False )