Kaggle competition: Mechanisms of Action (MoA) Prediction
sample_seq = sub_train_[115][-1]
old_res_mask = np.ones(len(sub_train_[115][-1]))
old_response_to_index(sample_seq, old_res_mask )<define_variables>
warnings.filterwarnings('ignore' )
class RIIID_Train_Dataset(Dataset):
def __init__(self, group, max_seq = 100):
self.group = group
self.max_seq = max_seq
self.user_ids = []
for i, user_id in enumerate(group.index):
q = group[user_id][0]
if len(q)< 2:
continue
self.user_ids.append(user_id)
 def __len__(self):
return len(self.user_ids)
 def __getitem__(self, idx):
user_id = self.user_ids[idx]
ques_, task_, part_, diff_, elapse_, corr_, target_, old_res_ = self.group[user_id]
old_res_mask_ = np.ones(len(old_res_))
pred_mask_ = old_res_mask_
old_res_ = old_response_to_index(old_res_, old_res_mask_)
 seq_len = len(ques_)
 pos_ = np.arange(seq_len)
 ques = np.zeros(self.max_seq, dtype = int)- 1
task = np.zeros(self.max_seq, dtype = int)- 1
part = np.zeros(self.max_seq, dtype = int)
 diff = np.zeros(self.max_seq, dtype = float)
 elapse = np.zeros(self.max_seq, dtype = float)
 corr = np.zeros(self.max_seq, dtype = float)
 target = np.zeros(self.max_seq, dtype = int)
 old_res = np.zeros(self.max_seq, dtype = int)
 pred_mask = np.zeros(self.max_seq, dtype = int)
 if seq_len >= self.max_seq:
if seq_len > self.max_seq:
start_index = np.random.randint(seq_len - self.max_seq)
 ques = ques_[start_index:(start_index + self.max_seq)]
task = task_[start_index:(start_index + self.max_seq)]
part = part_[start_index:(start_index + self.max_seq)]
diff = diff_[start_index:(start_index + self.max_seq)]
elapse = elapse_[start_index:(start_index + self.max_seq)]
corr = corr_[start_index:(start_index + self.max_seq)]
target = target_[start_index:(start_index + self.max_seq)]
old_res = old_res_[start_index:(start_index + self.max_seq)]
pred_mask = pred_mask_[start_index:(start_index + self.max_seq)]
else:
ques = ques_[-self.max_seq:]
task = task_[-self.max_seq:]
part = part_[-self.max_seq:]
diff = diff_[-self.max_seq:]
elapse = elapse_[-self.max_seq:]
corr = corr_[-self.max_seq:]
target = target_[-self.max_seq:]
old_res = old_res_[-self.max_seq:]
pred_mask = pred_mask_[-self.max_seq:]
else:
ques[-seq_len:] = ques_
task[-seq_len:] = task_
part[-seq_len:] = part_
diff[-seq_len:] = diff_
elapse[-seq_len:] = elapse_
corr[-seq_len:] = corr_
target[-seq_len:] = target_
old_res[-seq_len:] = old_res_
pred_mask[-seq_len:] = pred_mask_
task_pos = position(task, bundle_ignore = False)
 e_pos = position(task)
 d_pos = np.concatenate(( [0], e_pos[:-1]))
return {
'user': user_id,
'content_id': torch.tensor(ques, dtype = torch.long)+ 1,
'task_container_id': torch.tensor(task, dtype = torch.long)+ 1,
'part_id': torch.tensor(part, dtype = torch.long),
'diff_id': torch.tensor(diff, dtype = torch.float32),
'prior_elapsed_time_id': torch.tensor(elapse, dtype = torch.float32),
'user_correctness_id': torch.tensor(corr, dtype = torch.float32),
'old_response_id': torch.tensor(old_res, dtype = torch.long),
'encoder_position_id': torch.tensor(e_pos, dtype = torch.long),
'decoder_position_id': torch.tensor(d_pos, dtype = torch.long),
'task_position_id': torch.tensor(task_pos, dtype = torch.long),
'prediction_mask': torch.tensor(pred_mask, dtype = torch.long),
'target': torch.tensor(target, dtype = torch.float32)
 }<define_variables>
def reduce_mem_usage(props):
start_mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage of properties dataframe is :",start_mem_usg," MB")
 NAlist = []
for col in props.columns:
if props[col].dtype != object:
IsInt = False
mx = props[col].max()
mn = props[col].min()
if not np.isfinite(props[col] ).all() :
NAlist.append(col)
 props[col].fillna(mn-1,inplace=True)
 asint = props[col].fillna(0 ).astype(np.int64)
 result =(props[col] - asint)
 result = result.sum()
if result > -0.01 and result < 0.01:
IsInt = True
if IsInt:
if mn >= 0:
if mx < 255:
props[col] = props[col].astype(np.uint8)
 elif mx < 65535:
props[col] = props[col].astype(np.uint16)
 elif mx < 4294967295:
props[col] = props[col].astype(np.uint32)
 else:
props[col] = props[col].astype(np.uint64)
 else:
if mn > np.iinfo(np.int8 ).min and mx < np.iinfo(np.int8 ).max:
props[col] = props[col].astype(np.int8)
 elif mn > np.iinfo(np.int16 ).min and mx < np.iinfo(np.int16 ).max:
props[col] = props[col].astype(np.int16)
 elif mn > np.iinfo(np.int32 ).min and mx < np.iinfo(np.int32 ).max:
props[col] = props[col].astype(np.int32)
 elif mn > np.iinfo(np.int64 ).min and mx < np.iinfo(np.int64 ).max:
props[col] = props[col].astype(np.int64)
 else:
props[col] = props[col].astype(np.float32)
 print("___MEMORY USAGE AFTER COMPLETION:___")
 mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage is: ",mem_usg," MB")
 print("This is ",100*mem_usg/start_mem_usg,"% of the initial size")
 return props, NAlist
def train_short_form_loader(feature_file,target_file,extra_target_file=None):

 train_features = pd.read_csv(feature_file)
 train_targets = pd.read_csv(target_file)
 train_features,_= reduce_mem_usage(train_features)
 train_targets,_ = reduce_mem_usage(train_targets)
 if extra_target_file is not None:
extra_targets = pd.read_csv(extra_target_file)
 extra_targets,_ = reduce_mem_usage(extra_targets)
 train_targets = pd.merge(train_targets,extra_targets,on ='sig_id')
 del extra_targets
targets = train_targets.columns[1:]
train_melt=train_targets.merge(train_features,how="left",on="sig_id")
 del train_features,train_targets
train_melt.set_index("sig_id",inplace=True)
 train_melt["cp_type"]= train_melt["cp_type"].astype('category')
 train_melt["cp_dose"]= train_melt["cp_dose"].astype('category')
 return train_melt , targets.to_list()
def test_short_form_loader(feature_file):

 train_features = pd.read_csv(feature_file)
 train_features,_= reduce_mem_usage(train_features)
 train_melt = train_features.copy()
del train_features
train_melt.set_index("sig_id",inplace=True)
 train_melt["cp_type"]= train_melt["cp_type"].astype('category')
 train_melt["cp_dose"]= train_melt["cp_dose"].astype('category')
 return train_melt
class RIIID_Valid_Dataset(Dataset):
def __init__(self, valid, train, max_seq = 100):
self.valid_df = valid
self.train_df = train
self.max_seq = max_seq
self.users = list(self.valid_df.index)
 self.len_valid = np.array([0] + data_len_calculator(self.valid_df ).tolist())
 def __len__(self):
return self.len_valid[-1]
def __getitem__(self, idx):
user_idx = np.argmax(self.len_valid[self.len_valid <= idx])
 user = self.users[user_idx]
user_seq_valid = self.valid_df[user]
if user in self.train_df.index:
user_seq_train = self.train_df[user]
else:
user_seq_train = None
begin_idx = self.len_valid[self.len_valid <= idx][-1]
idx_in_user_batch = idx - begin_idx 
(ques_, task_, part_, diff_, elapse_, corr_, target_, old_res_), old_res_mask_ = valid_2_sequence(user, user_seq_valid, idx_in_user_batch,
train_seqs = user_seq_train)
 old_res_ = old_response_to_index(old_res_, old_res_mask_, use_mask = False)
 seq_len = len(ques_)
 pos_ = np.arange(seq_len)
 ques = np.zeros(self.max_seq, dtype = int)- 1
task = np.zeros(self.max_seq, dtype = int)- 1
part = np.zeros(self.max_seq, dtype = int)
 diff = np.zeros(self.max_seq, dtype = float)
 elapse = np.zeros(self.max_seq, dtype = float)
 corr = np.zeros(self.max_seq, dtype = float)
 target = np.zeros(self.max_seq, dtype = int)
 old_res = np.zeros(self.max_seq, dtype = int)
 if seq_len >= self.max_seq:
ques = ques_[-self.max_seq:]
task = task_[-self.max_seq:]
part = part_[-self.max_seq:]
diff = diff_[-self.max_seq:]
elapse = elapse_[-self.max_seq:]
corr = corr_[-self.max_seq:]
target = target_[-self.max_seq:]
old_res = old_res_[-self.max_seq:]
else:
ques[-seq_len:] = ques_
task[-seq_len:] = task_
part[-seq_len:] = part_
diff[-seq_len:] = diff_
elapse[-seq_len:] = elapse_
corr[-seq_len:] = corr_
target[-seq_len:] = target_
old_res[-seq_len:] = old_res_
task_pos = position(task, pad_include = True, bundle_ignore = False)
 pred_mask = prediction_mask(task_pos)
 e_pos = position(task)
 d_pos = np.concatenate(( [0], e_pos[:-1]))
return {
'user': user,
'content_id': torch.tensor(ques, dtype = torch.long)+ 1,
'task_container_id': torch.tensor(task, dtype = torch.long)+ 1,
'part_id': torch.tensor(part, dtype = torch.long),
'diff_id': torch.tensor(diff, dtype = torch.float32),
'prior_elapsed_time_id': torch.tensor(elapse, dtype = torch.float32),
'user_correctness_id': torch.tensor(corr, dtype = torch.float32),
'old_response_id': torch.tensor(old_res, dtype = torch.long),
'encoder_position_id': torch.tensor(e_pos, dtype = torch.long),
'decoder_position_id': torch.tensor(d_pos, dtype = torch.long),
'task_position_id': torch.tensor(task_pos, dtype = torch.long),
'prediction_mask': torch.tensor(pred_mask, dtype = torch.long),
'target': torch.tensor(target, dtype = torch.float32)
 }<create_dataframe>
input_directory = '.. /input/lish-moa/'
train_dataset = RIIID_Train_Dataset(sub_train_, 10)
 train_dataloader = DataLoader(train_dataset, batch_size = 5, shuffle = False, num_workers = 4 )<create_dataframe>
train,target_cols = train_short_form_loader(input_directory +'train_features.csv',input_directory+'train_targets_scored.csv')
 test = test_short_form_loader(input_directory +"test_features.csv")
valid_dataset = RIIID_Valid_Dataset(valid_df, sub_train_, max_seq = 10)
 valid_dataloader = DataLoader(valid_dataset, batch_size = 5, shuffle = False, num_workers = 0 )<normalization>
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
class ScheduledOptim() :

 def __init__(self, optimizer, d_model, n_warmup_steps):
self._optimizer = optimizer
self.n_warmup_steps = n_warmup_steps
self.n_current_steps = 0
self.init_lr = np.power(d_model, -0.5)
 def step_and_update_lr(self):
"Step with the inner optimizer"
self._update_learning_rate()
self._optimizer.step()
def zero_grad(self):
"Zero out the gradients by the inner optimizer"
self._optimizer.zero_grad()
def _get_lr_scale(self):
return np.min([
np.power(self.n_current_steps, -0.5),
np.power(self.n_warmup_steps, -1.5)* self.n_current_steps
])
 def _update_learning_rate(self):

 self.n_current_steps += 1
lr = self.init_lr * self._get_lr_scale()
for param_group in self._optimizer.param_groups:
param_group['lr'] = lr
class NoamOpt:
"Optim wrapper that implements rate."
def __init__(self, model_size, factor, warmup, optimizer):
self.optimizer = optimizer
self._step = 0
self.warmup = warmup
self.factor = factor
self.model_size = model_size
self._rate = 0
def zero_grad(self):
self.optimizer.zero_grad()
def step(self):
"Update parameters and rate"
self._step += 1
rate = self.rate()
for p in self.optimizer.param_groups:
p['lr'] = rate
self._rate = rate
self.optimizer.step()
def rate(self, step=None):
"Implement `lrate` above"
if step is None:
step = self._step
return self.factor * \ 
(self.model_size **(-0.5)*
min(step **(-0.5), step * self.warmup **(-1.5)))
 class NoamOptimizer:
def __init__(self, model, lr, model_size, warmup):
self._adam = torch.optim.Adam(model.parameters() , lr=lr)
 self._opt = NoamOpt(
 model_size=model_size, factor=1, warmup=warmup, optimizer=self._adam)
 def step(self, loss):
self._opt.zero_grad()
loss.backward()
self._opt.step()
def get_pad_mask(seq, pad_idx):
return(seq != pad_idx ).unsqueeze(-2)
 def get_subsequent_mask_3d(seq, only_before = True):
batch_size, seq_len = seq.shape
a = seq.unsqueeze(-1 ).expand(batch_size, seq_len, seq_len)
 b = seq.unsqueeze(1 ).expand(batch_size, seq_len, seq_len)+ int(only_before)
 return(a >= b)
 def get_subsequent_mask(seq):

 sz_b, len_s = seq.size()
subsequent_mask =(1 - torch.triu(torch.ones(( 1, len_s, len_s), device=seq.device), diagonal=1)).bool()
return subsequent_mask
def get_masks(seq, pad_idx, only_before = True):
encoder_mask =(get_pad_mask(seq, pad_idx)& get_subsequent_mask_3d(seq, only_before = only_before))
decoder_mask =(get_pad_mask(seq, pad_idx)& get_subsequent_mask(seq))
encoder_decoder_mask = encoder_mask
return encoder_mask, decoder_mask, encoder_decoder_mask
def clones(module, N):
"Produce N identical layers."
return nn.ModuleList([copy.deepcopy(module)for _ in range(N)] )<define_search_space>
from sklearn.decomposition import PCA
from sklearn.preprocessing import QuantileTransformer
from sklearn.compose import make_column_transformer,ColumnTransformer
pos = torch.tensor([[0,0,0,1,2,2,2,3,4,5],
[0,0,0,0,1,2,3,3,4,4]])
 get_subsequent_mask_3d(pos )<normalization>
def seed_everything(seed=42):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
get_pad_mask(pos, 0 )<normalization>
class CatIntMapper(BaseEstimator, TransformerMixin):
def __init__(self ,col,dicti):
self.col = col
self.dicti = dicti
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
assert X[self.col].isin(self.dicti.keys() ).all()
return pd.concat([X.drop(self.col,axis=1),X[self.col].map(self.dicti ).astype(int ).rename(self.col)],axis=1)
 def transform(self, X):
assert X[self.col].isin(self.dicti.keys() ).all()
return pd.concat([X.drop(self.col,axis=1),X[self.col].map(self.dicti ).astype(int ).rename(self.col)],axis=1)
 class NamedOutTWrapper(BaseEstimator, TransformerMixin):
def __init__(self,transformer,columns,inplace=False,prefix='_'):
self.transformer = transformer
self.cols = columns
self.inplace = inplace
self.prefix = prefix
self.transformer_name = self._get_transformer_name()
def fit(self, X, y = None):
self.transformer = self.transformer.fit(X[self.cols] , y)
 return self
def fit_transform(self, X, y = None):
transformed_columns = self.transformer.fit_transform(X[self.cols] , y)
 out=pd.DataFrame(index=X.index)
 if self.inplace:
out = X[self.cols]
out[self.cols] = transformed_columns
return pd.concat([X.drop(self.cols,axis=1),out],axis=1)
 else:
for i,values in enumerate(transformed_columns.transpose()):
out[ self.transformer_name + self.prefix + str(i)] = values
return pd.concat([X,out],axis=1)
 def transform(self, X):
transformed_columns = self.transformer.transform(X[self.cols])
 out=pd.DataFrame(index=X.index)
 if self.inplace:
out = X[self.cols]
out[self.cols] = transformed_columns
return pd.concat([X.drop(self.cols,axis=1),out],axis=1)
 else:
for i,values in enumerate(transformed_columns.transpose()):
out[ self.transformer_name + self.prefix + str(i)] = values
return pd.concat([X,out],axis=1)
 def _get_transformer_name(self):
return str(self.transformer.__class__ ).split('.')[-1][0:-2]
class IdentityTransformer:

 def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X
def transform(self, X):
return X
class SuppressControls(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X.loc[X['cp_type']!='ctl_vehicle'].drop('cp_type', axis=1)
 def transform(self, X):
return X.loc[X['cp_type']!='ctl_vehicle'].drop('cp_type', axis=1)
get_masks(pos, pad_idx = 0 )<compute_train_metric>
def multifold_indexer(train,target_columns,n_splits=10,random_state=12347,**kwargs):
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=n_splits,random_state=random_state,**kwargs)
 folds[ 'kfold']=0
for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=train[target_columns])) :
folds.iloc[v_idx,-1] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 return folds
def attention(query, key, value, mask=None, dropout=None):
"Compute 'Scaled Dot Product Attention'"
d_k = query.size(-1)
 scores = torch.matmul(query, key.transpose(-2, -1)) \
/ math.sqrt(d_k)
 if mask is not None:
scores = scores.masked_fill(mask == 0, -1e9)
 p_attn = F.softmax(scores, dim=-1)
 if dropout is not None:
p_attn = dropout(p_attn)
 return torch.matmul(p_attn, value), p_attn
class MultiHeadedAttention(nn.Module):
def __init__(self, h, d_model, dropout=0.1):
"Take in model size and number of heads."
super(MultiHeadedAttention, self ).__init__()
assert d_model % h == 0
self.d_k = d_model // h
self.h = h
self.linears = clones(nn.Linear(d_model, d_model, bias=False), 4)
 self.attn = None
self.dropout = nn.Dropout(p=dropout)
 def forward(self, query, key, value, mask=None):
"Implements Figure 2"
if mask is not None:
mask = mask.unsqueeze(1)
 nbatches = query.size(0)
 query, key, value = \
[l(x ).view(nbatches, -1, self.h, self.d_k ).transpose(1, 2)
 for l, x in zip(self.linears,(query, key, value)) ]
x, self.attn = attention(query, key, value, mask=mask,
dropout=self.dropout)
 x = x.transpose(1, 2 ).contiguous() \ 
.view(nbatches, -1, self.h * self.d_k)
 return self.linears[-1](x)
 class PositionwiseFeedForward(nn.Module):
"Implements FFN equation."
def __init__(self, d_model, d_ff, dropout=0.1):
super(PositionwiseFeedForward, self ).__init__()
self.w_1 = nn.Linear(d_model, d_ff)
 self.w_2 = nn.Linear(d_ff, d_model)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
return self.w_2(self.dropout(F.relu(self.w_1(x))))
class SAINTLayer_encoder(nn.Module):

 def __init__(self, hidden_dim, num_head, dropout):
super().__init__()
self._self_attn = MultiHeadedAttention(num_head, hidden_dim, dropout)
 self._ffn = PositionwiseFeedForward(hidden_dim, hidden_dim, dropout)
 self._layernorms = clones(nn.LayerNorm(hidden_dim, eps=1e-6), 2)
 self._dropout = nn.Dropout(dropout)
 def forward(self, src, mask = None):

 src2 = self._self_attn(query=src, key=src, value=src, mask=mask)
 src = src + self._dropout(src2)
 src = self._layernorms[0](src)
 src2 = self._ffn(src)
 src = src + self._dropout(src2)
 src = self._layernorms[1](src)
 return src
class SAINTLayer_decoder(nn.Module):

 def __init__(self, hidden_dim, num_head, dropout):
super().__init__()
self._self_attn_decoder = MultiHeadedAttention(num_head, hidden_dim, dropout)
 self._self_attn_encoder_decoder = MultiHeadedAttention(num_head, hidden_dim, dropout)
 self._ffn = PositionwiseFeedForward(hidden_dim, hidden_dim, dropout)
 self._layernorms = clones(nn.LayerNorm(hidden_dim, eps=1e-6), 3)
 self._dropout = nn.Dropout(dropout)
 def forward(self, tgt, memory, encoder_decoder_mask = None, decoder_mask = None):

 tgt2 = self._self_attn_decoder(query=tgt, key=tgt, value=tgt, mask=decoder_mask)
 tgt = tgt + self._dropout(tgt2)
 tgt = self._layernorms[0](tgt)
 tgt2 = self._self_attn_encoder_decoder(query=tgt, key=memory, value=memory, mask=encoder_decoder_mask)
 tgt = tgt + self._dropout(tgt2)
 tgt = self._layernorms[1](tgt)
 tgt2 = self._ffn(tgt)
 tgt = tgt + self._dropout(tgt2)
 tgt = self._layernorms[2](tgt)
 return tgt<choose_model_class>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
class SAINT_plus(nn.Module):
def __init__(self, question_num, task_num, max_seq = 100, d_model = 128, nhead = 8, dropout = 0.1, num_layers = 2):
super().__init__()
self.question_num = question_num
self.question_num = task_num
self.max_seq = max_seq
self.d_model = d_model
self.nhead = nhead
self.dropout = dropout
self._positional_embedding = nn.Embedding(self.max_seq + 1, d_model)
 self._question_embedding = nn.Embedding(question_num + 1, d_model)
 self._task_embedding = nn.Embedding(task_num + 1, d_model)
 self._part_embedding = nn.Embedding(9, d_model)
 self._target_embedding = nn.Embedding(len(responses_total_tokens), d_model)
 self._diff_embedding = nn.Linear(1, d_model)
 self._elapse_embedding = nn.Linear(1, d_model)
 self._corr_embedding = nn.Linear(1, d_model)
 self.encoder_layers = clones(SAINTLayer_encoder(d_model, nhead, dropout), num_layers)
 self.decoder_layers = clones(SAINTLayer_decoder(d_model, nhead, dropout), num_layers)
 self._prediction = nn.Linear(d_model, 1)
 def forward(self, question, task, part, diff, elapse, corr, target, e_pos, d_pos, task_pos):
device = question.device
diff = self._diff_embedding(diff.unsqueeze(-1))
elapse = self._elapse_embedding(elapse.unsqueeze(-1))
corr = self._corr_embedding(corr.unsqueeze(-1))
encoder_mask, decoder_mask, encoder_decoder_mask = get_masks(task_pos, 0, only_before = False)
 encoder_mask = encoder_mask.to(device)
 decoder_mask = decoder_mask.to(device)
 encoder_decoder_mask = encoder_decoder_mask.to(device)
 e_pos = self._positional_embedding(e_pos)
 d_pos = self._positional_embedding(d_pos)
 question = self._question_embedding(question)
 task = self._task_embedding(task)
 part = self._part_embedding(part)
 target = self._target_embedding(target)
 encoder = question + task + part + e_pos + diff
decoder = target + elapse + d_pos + corr
for layer in self.encoder_layers:
encoder = layer(encoder, mask = encoder_mask)
 for layer in self.decoder_layers:
decoder = layer(decoder, encoder, encoder_decoder_mask = encoder_decoder_mask,
decoder_mask = decoder_mask)
 output = self._prediction(decoder)
 return output.squeeze(-1 )<categorify>
class DAE_Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size=1100,hidden_size2=1300):
super(DAE_Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size2))
self.batch_norm3 = nn.BatchNorm1d(hidden_size2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size2, hidden_size))
self.dense4 = nn.utils.weight_norm(nn.Linear(hidden_size, num_features))
def forward(self, x,mode='DAE'):
x1 = F.relu(self.dense1(x))
x2 = self.batch_norm2(x1)
 x2 = F.relu(self.dense2(x2))
x3 = self.batch_norm3(x2)
 x3 = F.relu(self.dense3(x3))
out = self.dense4(x3)
 if mode == 'DAE':
return out
else:
return x1,x2,x3
def train_epoch(model, train_iterator, criterion, optim, device = 'cpu'):
model.train()
train_loss = []
num_corrects = 0
num_total = 0
labels = []
outs = []
masks = []
tbar = tqdm(train_iterator)
 for item in tbar:
x_ques = item['content_id'].to(device)
 x_task = item['task_container_id'].to(device)
 x_part = item['part_id'].to(device)
 x_diff = item['diff_id'].to(device)
 x_elapse = item['prior_elapsed_time_id'].to(device)
 x_corr = item['user_correctness_id'].to(device)
 x_ans = item['old_response_id'].to(device)
 x_e_pos = item['encoder_position_id'].to(device)
 x_d_pos = item['decoder_position_id'].to(device)
 x_task_pos = item['task_position_id'].to(device)
 mask = item['prediction_mask'].to(device)
 label = item['target'].to(device)
 with torch.set_grad_enabled(True):
output = model(x_ques, x_task, x_part, x_diff, x_elapse, x_corr, x_ans, x_e_pos, x_d_pos, x_task_pos)
 output = torch.masked_select(output, mask.bool())
 label = torch.masked_select(label, mask.bool())
 loss = criterion(output, label)
 optim.step(loss)
 train_loss.append(loss.item())
 pred =(torch.sigmoid(output)>= 0.5 ).long()
num_corrects +=(pred == label ).sum().item()
num_total += mask.sum().item()
labels.extend(label.view(-1 ).data.cpu().numpy())
 outs.extend(torch.sigmoid(output ).view(-1 ).data.cpu().numpy())
 batch_auc = roc_auc_score(label.view(-1 ).data.cpu().numpy() , torch.sigmoid(output ).view(-1 ).data.cpu().numpy())
 tbar.set_description('loss - {:.4f} || auc - {:.4f}'.format(loss, batch_auc))
acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
 loss = np.mean(train_loss)
 return loss, acc, auc<categorify>
map_controls = CatIntMapper('cp_type',{'ctl_vehicle': 0, 'trt_cp': 1})
 map_dose = CatIntMapper('cp_dose',{'D1': 1, 'D2': 0})
 map_time = CatIntMapper('cp_time',{24: 0, 48: 1, 72: 2})
 train = pd.read_csv(f'{input_directory}/train_features.csv')
 GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
GENES
Rankg_g_tansform = NamedOutTWrapper(QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal"),columns= GENES+CELLS,inplace=True)
 PCA_g_tansform = NamedOutTWrapper(PCA(20),columns= GENES,prefix ='_g')
 PCA_c_tansform = NamedOutTWrapper(PCA(20),columns= CELLS,prefix ='_c')
 class ColumnDropper(BaseEstimator, TransformerMixin):
def __init__(self, cols):
self.cols=cols
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
return X.drop(self.cols,axis=1)
 CatDropper =ColumnDropper(cols=['cp_type','cp_time','cp_dose'])
 transformers_list=[map_controls,map_dose,map_time,Rankg_g_tansform,CatDropper]
def valid_epoch(model, valid_iterator, criterion, device = 'cpu'):
model.eval()
valid_loss = []
num_corrects = 0
num_total = 0
labels = []
outs = []
masks = []
tbar = tqdm(valid_iterator)
 for item in tbar:
x_ques = item['content_id'].to(device)
 x_task = item['task_container_id'].to(device)
 x_part = item['part_id'].to(device)
 x_diff = item['diff_id'].to(device)
 x_elapse = item['prior_elapsed_time_id'].to(device)
 x_corr = item['user_correctness_id'].to(device)
 x_ans = item['old_response_id'].to(device)
 x_e_pos = item['encoder_position_id'].to(device)
 x_d_pos = item['decoder_position_id'].to(device)
 x_task_pos = item['task_position_id'].to(device)
 mask = item['prediction_mask'].to(device)
 label = item['target'].to(device)
 with torch.no_grad() :
output = model(x_ques, x_task, x_part, x_diff, x_elapse, x_corr, x_ans, x_e_pos, x_d_pos, x_task_pos)
 output = torch.masked_select(output, mask.bool())
 label = torch.masked_select(label, mask.bool())
 loss = criterion(output, label)
 valid_loss.append(loss.item())
 pred =(torch.sigmoid(output)>= 0.5 ).long()
num_corrects +=(pred == label ).sum().item()
num_total += mask.sum().item()
labels.extend(label.view(-1 ).data.cpu().numpy())
 outs.extend(torch.sigmoid(output ).view(-1 ).data.cpu().numpy())
 batch_auc = roc_auc_score(label.view(-1 ).data.cpu().numpy() , torch.sigmoid(output ).view(-1 ).data.cpu().numpy())
 tbar.set_description('loss - {:.4f} || auc - {:.4f}'.format(loss, batch_auc))
acc = num_corrects / num_total
auc = roc_auc_score(labels, outs)
 loss = np.mean(valid_loss)
 return loss, acc, auc<set_options>
exp_name = 'test_DAE_0.2_all_together'
def run_inference(X_train,y_train,X_valid,y_valid,X_test,fold, seed,inference_only=False,**kwargs):
seed_everything(seed)
 if not inference_only:
train_dataset = MoADataset(X_train, y_train)
 valid_dataset = MoADataset(X_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 testdataset = TestDataset(X_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = DAE_Model(
 num_features= X_train.shape[1] ,
num_targets= X_train.shape[1],
**kwargs 
)
 model.load_state_dict(torch.load(f".. /input/swap-dae-noise0-2/FOLD{fold}_{exp_name}.pth",map_location=torch.device('cpu')))
 model.to(DEVICE)
 if not inference_only:
oof = inference_infer_features_fn(model, validloader, DEVICE)
 else:
oof= 0
predictions = infer_features_fn(model, testloader, DEVICE)
 predictions = predictions
return oof, predictions
transformers_list
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 1000
BATCH_SIZE = 640
LEARNING_RATE = 2e-3
WEIGHT_DECAY = 1e-8
NFOLDS = 10
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
GAMMA=0.5
FACTOR=0.75
hidden_size=1100
hidden_size2=1300
PATIENCE=10
THRESHOLD = 5e-3
def infer_features_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs,mode='get_features')
 preds.append(torch.cat(outputs,axis=1 ).detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class train_config:
METRIC_ = 'max'
EPOCHS = 30
LR = 1e-5
WARM_UP = 4000
MODE = 'inference'
WORKERS = multiprocessing.cpu_count()
BATCH_SIZE = 128
TRAIN_RATIO = 0.97
CRITERION = nn.BCEWithLogitsLoss().to(DEVICE)
 D_MODEL = 128
N_HEADS = 8
N_LAYERS = 4
N_QUES = 13523
DROP = 0.1
if torch.cuda.is_available() :
MAP_LOCATION = lambda storage, loc: storage.cuda()
else:
MAP_LOCATION = 'cpu'<choose_model_class>
SEED = [0]
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 test = test_short_form_loader(".. /input/lish-moa/test_features.csv")
 train = pd.concat([train,test])
 train[target_cols]= train[target_cols].fillna(0)
 test = train.copy()
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
train = multifold_indexer(train,target_cols,n_splits=NFOLDS)
 for fold in range(NFOLDS):
train_df = train[train['kfold'] != fold]
valid_df = train[train['kfold'] == fold]
feature_cols = [col for col in train_df.columns if not(col in target_cols+['kfold'])]
pipeline_val = make_pipeline(*transformers_list)
 X_train, y_train = train_df[feature_cols], train_df[target_cols]
X_valid, y_valid = valid_df[feature_cols], valid_df[target_cols].values
X_train = pipeline_val.fit_transform(X_train)
 X_train = X_train.values
X_valid = pipeline_val.transform(X_valid)
 valid_index = X_valid.index
X_valid = X_valid.values
y_train = y_train.values
X_test = test[feature_cols]
X_test = pipeline_val.transform(X_test ).values
pred_ = run_inference(X_train,y_train,X_valid,y_valid,X_test,fold, seed,inference_only=True)
 break
if train_config.MODE == 'train':
auc_max = -np.inf
history = []
es = 0
model = SAINT_plus(train_config.N_QUES, 10000, max_seq = MAX_SEQ, d_model = train_config.D_MODEL,
nhead = train_config.N_HEADS, dropout = train_config.DROP, num_layers = train_config.N_LAYERS ).to(DEVICE)
 optimizer = NoamOptimizer(model, train_config.LR, train_config.D_MODEL, train_config.WARM_UP)
 valid_dataset = RIIID_Valid_Dataset(valid_df.sample(frac = 0.5, random_state = 2020, replace = False), sub_train_, max_seq = MAX_SEQ)
 valid_dataloader = DataLoader(valid_dataset, batch_size = train_config.BATCH_SIZE * 4,
shuffle = False, num_workers = train_config.WORKERS)
 for epoch in range(1, train_config.EPOCHS + 1):
weights = sub_train_.apply(lambda x: max(min(500, len(x[0])) , 5))
train_df_ = sub_train_.sample(frac = train_config.TRAIN_RATIO, weights = weights, replace = True,
random_state = epoch)
 train_dataset = RIIID_Train_Dataset(train_df_.reset_index(drop = True), MAX_SEQ)
 train_dataloader = DataLoader(train_dataset, batch_size = train_config.BATCH_SIZE,
shuffle = True, num_workers = train_config.WORKERS)
 train_loss, train_acc, train_auc = train_epoch(model, train_dataloader, train_config.CRITERION, optimizer, device = DEVICE)
 print("
Epoch
valid_loss, valid_acc, valid_auc = valid_epoch(model, valid_dataloader, train_config.CRITERION, device = DEVICE)
 print("Epoch
lr = optimizer._adam.param_groups[0]['lr']
history.append({"epoch": epoch, "lr": lr, **{"train_auc": train_auc, "train_acc": train_acc}, **{"valid_auc": valid_auc, "valid_acc": valid_acc}})
 if valid_auc > auc_max:
print("Epoch
auc_max = valid_auc
torch.save(model.state_dict() , MODEL_BEST)
 es = 0
else:
es += 1
if es > 20:
break
if history:
metric = 'auc'
history_pd = pd.DataFrame(history[1:] ).set_index("epoch")
 train_history_pd = history_pd[[c for c in history_pd.columns if "train_" in c]]
valid_history_pd = history_pd[[c for c in history_pd.columns if "valid_" in c]]
lr_history_pd = history_pd[[c for c in history_pd.columns if "lr" in c]]
fig, ax = plt.subplots(1,2, figsize =(DEFAULT_FIG_WIDTH, 6))
t_epoch = train_history_pd["train_%s" % metric].argmin() if train_config.METRIC_ == "min" else train_history_pd["train_%s" % metric].argmax()
v_epoch = valid_history_pd["valid_%s" % metric].argmin() if train_config.METRIC_ == "min" else valid_history_pd["valid_%s" % metric].argmax()
d = train_history_pd.plot(kind = "line", ax = ax[0], title = "Epoch: %d, Train: %.3f" %(t_epoch, train_history_pd.iloc[t_epoch,:]["train_%s" % metric]))
d = lr_history_pd.plot(kind = "line", ax = ax[0], secondary_y = True)
 d = valid_history_pd.plot(kind = "line", ax = ax[1], title = "Epoch: %d, Valid: %.3f" %(v_epoch, valid_history_pd.iloc[v_epoch,:]["valid_%s" % metric]))
d = lr_history_pd.plot(kind = "line", ax = ax[1], secondary_y = True)
 plt.savefig("train.png", bbox_inches = 'tight')
 plt.show()<groupby>
transformed_features = pd.DataFrame(pred_[1],index=test.index )
class RIIID_Test_Dataset(Dataset):
def __init__(self, group, test_df, max_seq = 100):
self.group = group
self.test_df = test_df
self.max_seq = max_seq
self.pos = np.cumsum([False] + [i != j for i, j in zip(test_df[USER_ID].values, test_df[USER_ID].values[1:])])
 def __len__(self):
return len(np.unique(self.pos))
def __getitem__(self, idx):
user_bundle = self.test_df.iloc[self.pos == idx]
user_id = user_bundle.user_id.unique() [0]
new_ques_ = user_bundle[CONTENT_ID].values
new_task_ = user_bundle[TASK_CONTAINER_ID].values
new_part_ = user_bundle[PART].values
new_diff_ = user_bundle[DIFF].values
new_elapse_ = user_bundle[ELAPSE].values
new_corr_ = user_bundle[CORR].values
new_seq_len = len(new_ques_)
 ques = np.zeros(self.max_seq, dtype = int)- 1
task = np.zeros(self.max_seq, dtype = int)- 1
part = np.zeros(self.max_seq, dtype = int)
 diff = np.zeros(self.max_seq, dtype = float)
 elapse = np.zeros(self.max_seq, dtype = float)
 corr = np.zeros(self.max_seq, dtype = float)
 old_res = np.zeros(self.max_seq, dtype = int)
 pred_mask = np.zeros(self.max_seq, dtype = int)
 if user_id in self.group.index:
ques_, task_, part_, diff_, elapse_, corr_, target_, old_res_ = self.group[user_id]
ques_ = np.append(ques_, new_ques_)
 task_ = np.append(task_, new_task_)
 part_ = np.append(part_, new_part_)
 diff_ = np.append(diff_, new_diff_)
 elapse_ = np.append(elapse_, new_elapse_)
 corr_ = np.append(corr_, new_corr_)
 old_res_ = np.append(old_res_, target_[-1] * np.ones(new_seq_len))
old_res_mask_ = np.ones(len(old_res_))
old_res_ = old_response_to_index(old_res_, old_res_mask_, use_mask = False)
 seq_len = len(ques_)
 if seq_len >= self.max_seq:
ques = ques_[-self.max_seq:]
task = task_[-self.max_seq:]
part = part_[-self.max_seq:]
diff = diff_[-self.max_seq:]
elapse = elapse_[-self.max_seq:]
corr = corr_[-self.max_seq:]
old_res = old_res_[-self.max_seq:]
else:
ques[-seq_len:] = ques_
task[-seq_len:] = task_
part[-seq_len:] = part_
diff[-seq_len:] = diff_
elapse[-seq_len:] = elapse_
corr[-seq_len:] = corr_
old_res[-seq_len:] = old_res_
else:
ques = np.append(ques, new_ques_)
 task = np.append(task, new_task_)
 part = np.append(part, new_part_)
 diff = np.append(diff, new_diff_)
 elapse = np.append(elapse, new_elapse_)
 corr = np.append(corr, new_corr_)
 if new_seq_len == 1:
old_res = np.append(old_res, START_ID)
 else:
old_res = np.concatenate(( old_res, [START_ID], [MASK_ID] *(new_seq_len - 1)))
 ques = ques[-self.max_seq:]
task = task[-self.max_seq:]
part = part[-self.max_seq:]
diff = diff[-self.max_seq:]
elapse = elapse[-self.max_seq:]
corr = corr[-self.max_seq:]
old_res = old_res[-self.max_seq:]
task_pos = position(task, pad_include = True, bundle_ignore = False)
 pred_mask[-new_seq_len:] = np.ones(new_seq_len)
 e_pos = position(task)
 d_pos = np.concatenate(( [0], e_pos[:-1]))
return {
'user': user_id,
'content_id': torch.tensor(ques, dtype = torch.long)+ 1,
'task_container_id': torch.tensor(task, dtype = torch.long)+ 1,
'part_id': torch.tensor(part, dtype = torch.long),
'diff_id': torch.tensor(diff, dtype = torch.float32),
'prior_elapsed_time_id': torch.tensor(elapse, dtype = torch.float32),
'user_correctness_id': torch.tensor(corr, dtype = torch.float32),
'old_response_id': torch.tensor(old_res, dtype = torch.long),
'encoder_position_id': torch.tensor(e_pos, dtype = torch.long),
'decoder_position_id': torch.tensor(d_pos, dtype = torch.long),
'task_position_id': torch.tensor(task_pos, dtype = torch.long),
'prediction_mask': torch.tensor(pred_mask, dtype = torch.long)
 }<load_pretrained>
class DAE_Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size=1100,hidden_size2=1300):
super(DAE_Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size2))
self.batch_norm3 = nn.BatchNorm1d(hidden_size2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size2, hidden_size))
self.dense4 = nn.utils.weight_norm(nn.Linear(hidden_size, num_features))
def forward(self, x,mode='DAE'):
x1 = F.relu(self.dense1(x))
x2 = self.batch_norm2(x1)
 x2 = F.relu(self.dense2(x2))
x3 = self.batch_norm3(x2)
 x3 = F.relu(self.dense3(x3))
out = self.dense4(x3)
 if mode == 'DAE':
return out
else:
return x1,x2,x3
model = SAINT_plus(train_config.N_QUES, 10000, max_seq = MAX_SEQ, d_model = train_config.D_MODEL,
nhead = train_config.N_HEADS, dropout = train_config.DROP, num_layers = train_config.N_LAYERS ).to(DEVICE)
 if train_config.MODE == 'train':
resume_path = MODEL_BEST
elif train_config.MODE == 'inference':
resume_path = r'.. /input/riiid-saint-best-models/model_best_user_correctness_included_v5.pt'
model.load_state_dict(torch.load(resume_path, map_location = train_config.MAP_LOCATION))
model.to(DEVICE)
 model.eval()<load_pretrained>
from sklearn.decomposition import PCA
from sklearn.preprocessing import QuantileTransformer
from sklearn.pipeline import make_pipeline,make_union
EMULATION = False
if EMULATION:
target_val = pd.read_pickle('.. /input/riiid-cross-validation-files/cv1_valid.pickle' )<feature_engineering>
map_controls = CatIntMapper('cp_type',{'ctl_vehicle': 0, 'trt_cp': 1})
 map_dose = CatIntMapper('cp_dose',{'D1': 1, 'D2': 0})
 map_time = CatIntMapper('cp_time',{24: 0, 48: 1, 72: 2})
class Iter_Valid(object):
def __init__(self, df, max_user=1000):
df = df.reset_index(drop=True)
 self.df = df
self.user_answer = df['user_answer'].astype(str ).values
self.answered_correctly = df['answered_correctly'].astype(str ).values
df['prior_group_responses'] = "[]"
df['prior_group_answers_correct'] = "[]"
self.sample_df = df[df['content_type_id'] == 0][['row_id']]
self.sample_df['answered_correctly'] = 0
self.len = len(df)
 self.user_id = df.user_id.values
self.task_container_id = df.task_container_id.values
self.content_type_id = df.content_type_id.values
self.max_user = max_user
self.current = 0
self.pre_user_answer_list = []
self.pre_answered_correctly_list = []
def __iter__(self):
return self
def fix_df(self, user_answer_list, answered_correctly_list, pre_start):
df= self.df[pre_start:self.current].copy()
sample_df = self.sample_df[pre_start:self.current].copy()
df.loc[pre_start,'prior_group_responses'] = '[' + ",".join(self.pre_user_answer_list)+ ']'
df.loc[pre_start,'prior_group_answers_correct'] = '[' + ",".join(self.pre_answered_correctly_list)+ ']'
self.pre_user_answer_list = user_answer_list
self.pre_answered_correctly_list = answered_correctly_list
return df, sample_df
def __next__(self):
added_user = set()
pre_start = self.current
pre_added_user = -1
pre_task_container_id = -1
user_answer_list = []
answered_correctly_list = []
while self.current < self.len:
crr_user_id = self.user_id[self.current]
crr_task_container_id = self.task_container_id[self.current]
crr_content_type_id = self.content_type_id[self.current]
if crr_content_type_id == 1:
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
continue
if crr_user_id in added_user and(( crr_user_id != pre_added_user)or(crr_task_container_id != pre_task_container_id)) :
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 if len(added_user)== self.max_user:
if crr_user_id == pre_added_user and crr_task_container_id == pre_task_container_id:
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
continue
else:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 added_user.add(crr_user_id)
 pre_added_user = crr_user_id
pre_task_container_id = crr_task_container_id
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
if pre_start < self.current:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 else:
raise StopIteration()<predict_on_test>
train = pd.read_csv(f'{input_directory}/train_features.csv' )
if EMULATION:
iter_test = Iter_Valid(target_val, max_user = 1000)
 predicted = []
def set_predict(df):
predicted.append(df)
 else:
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict
test_train_group = train_group.copy()
prev_test_df = None<data_type_conversions>
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
user_sum_dict = user_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int32' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))<feature_engineering>
Rankg_g_tansform = NamedOutTWrapper(QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal"),columns= GENES+CELLS,inplace=True )
for ii,(test_df, sample_prediction_df)in enumerate(tqdm(iter_test)) :
print('*' * 50)
 if(prev_test_df is not None)&(psutil.virtual_memory().percent < 90):
print(psutil.virtual_memory().percent)
 prev_test_df[TARGET] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False].reset_index(drop = True)
 pos = np.cumsum([False] + [i != j for i, j in zip(prev_test_df[USER_ID].values, prev_test_df[USER_ID].values[1:])])
 content_ids = prev_test_df[CONTENT_ID].values
user_ids = prev_test_df[USER_ID].values
targets = prev_test_df[TARGET].values
for content_id, user_id, answered_correctly in zip(content_ids, user_ids, targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
content_sum_dict[content_id] += answered_correctly
content_count_dict[content_id] += 1
for prev_user_id in np.unique(pos):
sub_user_df = prev_test_df[pos == prev_user_id]
user = sub_user_df[USER_ID].unique() [0]
prev_group_content = sub_user_df[CONTENT_ID].values
prev_group_task = sub_user_df[TASK_CONTAINER_ID].values
prev_group_part = sub_user_df[PART].values
prev_group_diff = sub_user_df[DIFF].values
prev_group_elapse = sub_user_df[ELAPSE].values
prev_group_correct = sub_user_df[CORR].values
prev_group_target = sub_user_df[TARGET].values
if user in test_train_group.index:
prev_group_old_res = np.concatenate(( [test_train_group[user][6][-1]], prev_group_target)) [:-1]
test_train_group[user] =(np.append(test_train_group[user][0], prev_group_content),
np.append(test_train_group[user][1], prev_group_task),
np.append(test_train_group[user][2], prev_group_part),
np.append(test_train_group[user][3], prev_group_diff),
np.append(test_train_group[user][4], prev_group_elapse),
np.append(test_train_group[user][5], prev_group_correct),
np.append(test_train_group[user][6], prev_group_target),
np.append(test_train_group[user][7], prev_group_old_res))
else:
prev_group_old_res = np.array([1] * len(prev_group_content))
test_train_group[user] =(prev_group_content, prev_group_task, prev_group_part,
prev_group_diff, prev_group_elapse, prev_group_correct,
prev_group_target, prev_group_old_res)
 if len(test_train_group[user][0])> MAX_SEQ:
new_group_content = test_train_group[user][0][-MAX_SEQ:]
new_group_task = test_train_group[user][1][-MAX_SEQ:]
new_group_part = test_train_group[user][2][-MAX_SEQ:]
new_group_diff = test_train_group[user][3][-MAX_SEQ:]
new_group_elapse = test_train_group[user][4][-MAX_SEQ:]
new_group_correct = test_train_group[user][5][-MAX_SEQ:]
new_group_target = test_train_group[user][6][-MAX_SEQ:]
new_group_old_res = test_train_group[user][7][-MAX_SEQ:]
test_train_group[user] =(new_group_content, new_group_task, new_group_part,
new_group_diff, new_group_elapse, new_group_correct,
new_group_target, new_group_old_res)
 test_df = pd.merge(test_df, questions_df, left_on = CONTENT_ID, right_on = 'question_id', how = 'left')
 user_sum = np.zeros(len(test_df), dtype = np.int16)
 user_count = np.zeros(len(test_df), dtype = np.int16)
 content_sum = np.zeros(len(test_df), dtype = np.int32)
 content_count = np.zeros(len(test_df), dtype = np.int32)
 for i,(user_id, content_id)in enumerate(zip(test_df[USER_ID].values, test_df[CONTENT_ID].values)) :
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
test_df[DIFF] = content_sum / content_count
test_df[CORR] = user_sum / user_count
test_df[ELAPSE] = test_df[ELAPSE] / 300000 * 100
test_df.fillna(0, inplace = True)
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False].reset_index(drop = True)
 test_dataset = RIIID_Test_Dataset(test_train_group, test_df)
 test_dataloader = DataLoader(test_dataset, batch_size = train_config.BATCH_SIZE, shuffle = False, drop_last = False)
 outs = []
for item in test_dataloader:
x_ques = item['content_id'].to(DEVICE)
 x_task = item['task_container_id'].to(DEVICE)
 x_part = item['part_id'].to(DEVICE)
 x_diff = item['diff_id'].to(DEVICE)
 x_elapse = item['prior_elapsed_time_id'].to(DEVICE)
 x_corr = item['user_correctness_id'].to(DEVICE)
 x_ans = item['old_response_id'].to(DEVICE)
 x_e_pos = item['encoder_position_id'].to(DEVICE)
 x_d_pos = item['decoder_position_id'].to(DEVICE)
 x_task_pos = item['task_position_id'].to(DEVICE)
 mask = item['prediction_mask'].to(DEVICE)
 with torch.no_grad() :
output = model(x_ques, x_task, x_part, x_diff, x_elapse, x_corr, x_ans, x_e_pos, x_d_pos, x_task_pos)
 output = torch.masked_select(output, mask.bool())
 outs.extend(torch.sigmoid(output ).view(-1 ).data.cpu().numpy())
 test_df['answered_correctly'] = outs
set_predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<load_from_csv>
PCA_g_tansform = NamedOutTWrapper(PCA(20),columns= GENES,prefix ='_g' )
%%time
print('Starting...')
 train_df = pd.read_csv('.. /input/riiid-test-answer-prediction/train.csv', usecols=['timestamp', 'user_id', 'content_id', 'content_type_id', 'task_container_id', 'user_answer', 'prior_question_elapsed_time', 'prior_question_had_explanation',
'answered_correctly',
],
dtype={'timestamp': 'int64', 'user_id': 'int32', 'content_id': 'int16', 'content_type_id': 'int8',
'task_container_id': 'int16', 'user_answer': 'int8', 'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean',
'answered_correctly': 'int8',
} 
)
 print('Sorting...')
 print('Filling...')
 train_df['prior_question_had_explanation'] = train_df['prior_question_had_explanation'].astype(np.float ).fillna(-1 ).astype(np.int8)
 print('Filling...')
 train_df['prior_question_elapsed_time'] = train_df['prior_question_elapsed_time'].fillna(-1000)
 print(train_df.head())
 print('Done' )<import_modules>
PCA_c_tansform = NamedOutTWrapper(PCA(20),columns= CELLS,prefix ='_c' )
<data_type_conversions>
class ColumnDropper(BaseEstimator, TransformerMixin):
def __init__(self, cols):
self.cols=cols
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
return X.drop(self.cols,axis=1)
%%time
gc.collect()
def compose_single_user_data(r):
timestamps = r['timestamp'].values
prev_timestamps = np.pad(timestamps[:-1],(1, 0), 'constant', constant_values=0 ).astype(np.int64)
 content_ids = r['content_id'].values
content_type_ids = r['content_type_id'].values
is_question =(content_type_ids == 0)
 question_timestamps = timestamps[is_question]
question_prev_timestamps = prev_timestamps[is_question]
question_ids = content_ids[is_question]
question_task_container_ids = r['task_container_id'].values[is_question]
user_answers = r['user_answer'].values[is_question]
prior_question_elapsed_times = r['prior_question_elapsed_time'].values[is_question]
prior_question_had_explanation_idxs = r['prior_question_had_explanation'].values[is_question] + 1
answered_correctly_idxs = r['answered_correctly'].values[is_question] + 1
prev_was_lecture = np.pad(content_type_ids[:-1],(1, 0), 'constant', constant_values=0 ).astype(np.int8)
 prev_prev_timestamps = np.pad(prev_timestamps[:-1],(1, 0), 'constant', constant_values=0 ).astype(np.int64)
 question_prev_was_lecture = prev_was_lecture[is_question]
question_prev_prev_timestamps = prev_prev_timestamps[is_question]
d = {
'question_timestamps': question_timestamps,
'question_prev_timestamps': question_prev_timestamps,
'question_ids': question_ids,
'question_task_container_ids': question_task_container_ids,
'user_answers': user_answers,
'prior_question_elapsed_times': prior_question_elapsed_times,
'prior_question_had_explanation_idxs': prior_question_had_explanation_idxs,
'answered_correctly_idxs': answered_correctly_idxs,
'question_prev_was_lecture': question_prev_was_lecture,
'question_prev_prev_timestamps': question_prev_prev_timestamps,
}
return d
print('Grouping...')
 train_df = train_df.groupby('user_id')
 print('Applying...')
 train_df = train_df.apply(compose_single_user_data)
 print('Done')
 gc.collect()<set_options>
CatDropper =ColumnDropper(cols=['cp_type','cp_time','cp_dose'] )
pd.set_option('display.max_rows',100)
 train_df.head(10 )<define_variables>
transformers_list=[map_controls,map_dose,map_time,Rankg_g_tansform,CatDropper]
user_idxs_by_ids = dict(zip(train_df.index, range(1, len(train_df)+ 1)))
 user_ids_by_idxs = np.array([-1] + list(train_df.index), dtype=np.int32)
 user_data_by_idxs = [None] + list(train_df)
 del train_df
user_data_questions_counts_by_idxs = [len(user_data['question_ids'])if user_data is not None else -1 for user_data in user_data_by_idxs]
user_id = 115
user_idx = user_idxs_by_ids[user_id]
print(f'user_id={user_id} user_idx={user_idx} user_ids_by_idxs[user_idxs_by_ids[user_id]]={user_ids_by_idxs[user_idxs_by_ids[user_id]]}')
 user_data = user_data_by_idxs[user_idx]
print(f'user_data={str(user_data)[:1000]}' )<load_pretrained>
exp_name = 'test_DAE_0.2_all_together'
<load_from_csv>
def run_inference(X_train,y_train,X_valid,y_valid,X_test,fold, seed,inference_only=False,**kwargs):
seed_everything(seed)
 if not inference_only:
train_dataset = MoADataset(X_train, y_train)
 valid_dataset = MoADataset(X_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 testdataset = TestDataset(X_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = DAE_Model(
 num_features= X_train.shape[1] ,
num_targets= X_train.shape[1],
**kwargs 
)
 model.load_state_dict(torch.load(f".. /input/swap-dae-noise0-2/FOLD{fold}_{exp_name}.pth",map_location=torch.device('cpu')))
 model.to(DEVICE)
 if not inference_only:
oof = inference_infer_features_fn(model, validloader, DEVICE)
 else:
oof= 0
predictions = infer_features_fn(model, testloader, DEVICE)
 predictions = predictions
return oof, predictions
questions_df = pd.read_csv('.. /input/riiid-test-answer-prediction/questions.csv',
dtype={'question_id': 'int16', 'bundle_id': 'int32', 'correct_answer': 'int8', 'part': 'int8', 'tags': 'object'})
 questions_df.fillna('', inplace=True)
 def extract_tags_list(x):
tag_idxs = [0, 0, 0, 0, 0, 0]
if type(x)is not str:
print('not str:', type(x), x)
 else:
if x != '':
for i, part in enumerate(x.split(' ')) :
tag = int(part)
 if tag < 0 or tag > 187:
print('bad:', tag, x)
 tag_idxs[i] = tag + 1
return tag_idxs
questions_df['tags_list'] = questions_df['tags'].apply(extract_tags_list)
 questions_df.head(10)
 print('unique question_ids:', len(questions_df['question_id'].unique()))
min_question_id = questions_df['question_id'].min()
print('min question_id:', min_question_id)
 max_question_id = questions_df['question_id'].max()
print('max question_id:', max_question_id)
 print('unique bundle_ids:', len(questions_df['bundle_id'].unique()))
print('min bundle_id:', questions_df['bundle_id'].min())
 print('max bundle_id:', questions_df['bundle_id'].max())
 print('unique correct_answers:', len(questions_df['correct_answer'].unique()))
print('min correct_answer:', questions_df['correct_answer'].min())
 print('max correct_answer:', questions_df['correct_answer'].max())
 print('unique parts:', len(questions_df['part'].unique()))
print('min part:', questions_df['part'].min())
 print('max part:', questions_df['part'].max())
 questions = questions_df.to_dict()
question_idxs_by_ids_dict = {}
question_idxs_by_ids = np.zeros(( max_question_id+1,), dtype=np.int16)
 for i, question_id in questions['question_id'].items() :
question_idxs_by_ids_dict[question_id] = i + 1
question_idxs_by_ids[question_id] = i + 1
question_ids_by_idxs = np.array([-1] + list(questions['question_id']), dtype=np.int16)
 question_bundle_ids_by_idxs = np.array([-1] + list(questions['bundle_id'].values()), dtype=np.int32)
 question_correct_answers_by_idxs = np.array([-1] + list(questions['correct_answer'].values()), dtype=np.int8)
 del questions
del questions_df
gc.collect()
question_id = 13522
question_idx = question_idxs_by_ids[question_id]
print(f'question_id={question_id} question_idx={question_idx} question_ids_by_idxs[question_idxs_by_ids[question_id]]={question_ids_by_idxs[question_idxs_by_ids[question_id]]}' )<randomize_order>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 1000
BATCH_SIZE = 640
LEARNING_RATE = 2e-3
WEIGHT_DECAY = 1e-8
NFOLDS = 10
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
GAMMA=0.5
FACTOR=0.75
hidden_size=1100
hidden_size2=1300
PATIENCE=10
THRESHOLD = 5e-3
nonzero_user_idxs = np.arange(1, len(user_ids_by_idxs), dtype=np.int32)
 print(f'nonzero_user_idxs: {len(nonzero_user_idxs)}')
 test_set_fraction = 0.1
test_set_user_idxs = nonzero_user_idxs.copy()
np.random.shuffle(test_set_user_idxs)
 test_set_user_idxs = np.array(sorted(test_set_user_idxs[:int(len(test_set_user_idxs)* test_set_fraction * 2)].tolist()))
test_set_user_idxs_set = set(test_set_user_idxs)
 print(f'test_set_user_idxs: {len(test_set_user_idxs)} {list(test_set_user_idxs)[:100]}')
 max_user_data_questions_count = 0
for user_idx in nonzero_user_idxs:
user_data_questions_count = user_data_questions_counts_by_idxs[user_idx]
if user_data_questions_count > max_user_data_questions_count:
max_user_data_questions_count = user_data_questions_count
print(f'max_user_data_questions_count: {max_user_data_questions_count}')
 def sample_test(max_seq_len):
test_set_start_positions_by_user_idxs = [-1]
test_user_idxs = []
test_start_poss = []
test_lengths = []
test_roc_auc_start_poss = []
for user_idx in nonzero_user_idxs:
user_data_questions_count = user_data_questions_counts_by_idxs[user_idx]
if user_idx in test_set_user_idxs_set:
test_set_start_position_by_user_idx = np.random.randint(low=0, high=user_data_questions_count)
 else:
test_set_start_position_by_user_idx = user_data_questions_count
test_set_start_positions_by_user_idxs.append(test_set_start_position_by_user_idx)
 if user_idx in test_set_user_idxs_set:
excess =(user_data_questions_count - test_set_start_position_by_user_idx)% max_seq_len
if excess > 0:
test_roc_auc_start_pos = test_set_start_position_by_user_idx
start_pos = test_roc_auc_start_pos + excess - max_seq_len
if start_pos < 0:
length = max_seq_len -(-start_pos)
 start_pos = 0
else:
length = max_seq_len
test_user_idxs.append(user_idx)
 test_start_poss.append(start_pos)
 test_roc_auc_start_poss.append(test_roc_auc_start_pos)
 assert length > 0
test_lengths.append(length)
 while start_pos < user_data_questions_count:
length = min(user_data_questions_count - start_pos, max_seq_len)
 test_user_idxs.append(user_idx)
 test_start_poss.append(start_pos)
 test_roc_auc_start_poss.append(start_pos)
 assert length > 0
test_lengths.append(length)
 start_pos += length
return test_set_start_positions_by_user_idxs, test_user_idxs, test_start_poss, test_lengths, test_roc_auc_start_poss
test_set_start_positions_by_user_idxs, test_user_idxs, test_start_poss, test_lengths, test_roc_auc_start_poss = sample_test(200)
 print(f'test_user_idxs: {len(test_user_idxs)} test_start_poss: {len(test_start_poss)} test_roc_auc_start_poss: {len(test_roc_auc_start_poss)} test_lengths: {len(test_lengths)}')
 def sample_train(max_seq_len):
train_user_idxs = []
train_start_poss = []
train_lengths = []
train_samples_counts_by_user_idxs = [-1]
for user_idx in nonzero_user_idxs:
user_data_questions_count = user_data_questions_counts_by_idxs[user_idx]
test_set_start_position_by_user_idx = test_set_start_positions_by_user_idxs[user_idx]
start_pos = 0
train_samples_count_by_user_idx = 0
small_amount_of_data_for_user =(test_set_start_position_by_user_idx <= max_seq_len)
 while start_pos < test_set_start_position_by_user_idx:
if(start_pos == 0):
high = min(test_set_start_position_by_user_idx, max_seq_len)
 if high == 1:
length = 1
else:
length = np.random.randint(low=1, high=high)
 else:
length = min(test_set_start_position_by_user_idx - start_pos, max_seq_len)
 if(length >= max_seq_len * 0.75)or(np.random.randint(low=0, high=100)>= 90):
train_samples_count_by_user_idx += 1
train_user_idxs.append(user_idx)
 train_start_poss.append(start_pos)
 assert length > 0
train_lengths.append(length)
 start_pos += length
train_samples_counts_by_user_idxs.append(train_samples_count_by_user_idx)
 train_indexes = np.arange(len(train_lengths))
np.random.shuffle(train_indexes)
 return train_user_idxs, train_start_poss, train_lengths, train_indexes, train_samples_counts_by_user_idxs
train_user_idxs, train_start_poss, train_lengths, train_indexes, train_samples_counts_by_user_idxs = sample_train(200)
 users_with_much_data_count = 0
for train_samples_count_by_user_idx in train_samples_counts_by_user_idxs:
if train_samples_count_by_user_idx >= 5:
users_with_much_data_count += 1
print(f'train_user_idxs: {len(train_user_idxs)} train_start_poss: {len(train_start_poss)} train_lengths: {len(train_lengths)} train_indexes: {len(train_indexes)} users_with_much_data_count: {users_with_much_data_count}')
 del train_user_idxs
del train_start_poss
del train_lengths
del train_indexes
del train_samples_counts_by_user_idxs
del test_set_start_positions_by_user_idxs
del test_user_idxs
del test_start_poss
del test_lengths
del test_roc_auc_start_poss<set_options>
def infer_features_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs,mode='get_features')
 preds.append(torch.cat(outputs,axis=1 ).detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
train_samples_counts_by_user_idxs = None
class RiiidDataset(Dataset):
def __init__(self, is_train_set=True, max_seq_len=128):
super(RiiidDataset, self ).__init__()
self.is_train_set = is_train_set
self.max_seq_len = max_seq_len
def prepare(self):
if self.is_train_set:
global train_samples_counts_by_user_idxs
self.set_user_idxs, self.set_start_poss, self.set_lengths, self.set_indexes, train_samples_counts_by_user_idxs_ = sample_train(self.max_seq_len)
 if train_samples_counts_by_user_idxs is None:
train_samples_counts_by_user_idxs = train_samples_counts_by_user_idxs_
self.set_roc_auc_start_poss = self.set_start_poss
else:
self.set_user_idxs, self.set_start_poss, self.set_lengths, self.set_roc_auc_start_poss = test_user_idxs, test_start_poss, test_lengths, test_roc_auc_start_poss
def __len__(self):
samples_count = len(self.set_lengths)
 return samples_count
def __getitem__(self, index):
if index < 0:
user_idx = -index
user_data = user_data_by_idxs[user_idx]
seq_len = len(user_data['question_ids'])
 if seq_len >= self.max_seq_len:
start_pos = seq_len - self.max_seq_len
seq_len = self.max_seq_len
else:
start_pos = 0
set_roc_auc_start_poss = start_pos
user_data_test_start_pos = 0
else:
if self.is_train_set:
index = self.set_indexes[index]
user_idx = self.set_user_idxs[index]
user_data = user_data_by_idxs[user_idx]
user_data_test_start_pos = test_set_start_positions_by_user_idxs[user_idx]
seq_len = self.set_lengths[index]
start_pos = self.set_start_poss[index]
set_roc_auc_start_poss = self.set_roc_auc_start_poss[index]
user_data_seq = user_data['question_timestamps']
question_timestamps = np.zeros(self.max_seq_len, dtype=user_data_seq.dtype)
 question_timestamps_without_padding = user_data_seq[start_pos:start_pos+seq_len]
question_timestamps[:seq_len] = question_timestamps_without_padding
user_data_seq = user_data['question_prev_timestamps']
question_from_prev_timestamps_idxs = np.zeros(self.max_seq_len, dtype=np.int8)
 question_prev_timestamps_without_padding = user_data_seq[start_pos:start_pos+seq_len]
question_from_prev_timestamps_idxs[:seq_len] = np.minimum(np.log(np.maximum(question_timestamps_without_padding - question_prev_timestamps_without_padding, 0)/ 1000.0 + 2.0)/ np.log(1.5), 31 ).astype(np.int8)
 if start_pos == 0:
question_from_prev_timestamps_idxs[0] = 0
user_data_seq = user_data['question_ids']
question_idxs = np.zeros(self.max_seq_len, dtype=np.int32)
 question_idxs_without_padding = question_idxs_by_ids[user_data_seq[start_pos:start_pos+seq_len]]
question_idxs[:seq_len] = question_idxs_without_padding
user_data_seq = user_data['question_prev_was_lecture']
question_prev_was_lecture_without_padding = user_data_seq[start_pos:start_pos+seq_len]
user_data_seq = user_data['question_prev_prev_timestamps']
question_prev_prev_timestamps_without_padding = user_data_seq[start_pos:start_pos+seq_len]
question_pre_lecture_length_idxs_without_padding = np.minimum(np.log(np.maximum(question_prev_timestamps_without_padding - question_prev_prev_timestamps_without_padding, 0)/ 1000.0 + 2.0)/ np.log(1.5), 31 ).astype(np.int8)
 question_pre_lecture_length_idxs_without_padding = question_pre_lecture_length_idxs_without_padding * question_prev_was_lecture_without_padding
question_pre_lecture_length_idxs = np.zeros(self.max_seq_len, dtype=np.int8)
 question_pre_lecture_length_idxs[:seq_len] = question_pre_lecture_length_idxs_without_padding
if start_pos == 0:
question_pre_lecture_length_idxs[0:2] = 0
elif start_pos == 1:
question_pre_lecture_length_idxs[0:1] = 0
user_data_seq = user_data['question_task_container_ids']
question_task_container_ids = np.zeros(self.max_seq_len, dtype=user_data_seq.dtype)
 question_task_container_ids_without_padding = user_data_seq[start_pos:start_pos+seq_len]
is_bundle_id_changed =(question_task_container_ids_without_padding != np.pad(question_task_container_ids_without_padding[:-1],(1, 0), 'constant', constant_values=-1)).astype(np.int32)
 local_bundle_ids_without_padding = np.cumsum(is_bundle_id_changed)
 local_bundle_ids = np.zeros(self.max_seq_len, dtype=np.int32)
 local_bundle_ids[:seq_len] = local_bundle_ids_without_padding
last_bundle_mask = np.zeros(self.max_seq_len, dtype=np.int8)
 last_bundle_id = local_bundle_ids_without_padding[-1]
k = len(local_bundle_ids_without_padding)- 1
while(k >= 0)and(local_bundle_ids_without_padding[k] == last_bundle_id):
last_bundle_mask[k] = 1
k -= 1
user_data_seq = user_data['prior_question_elapsed_times']
prior_question_elapsed_times_idxs = np.zeros(self.max_seq_len, dtype=np.int8)
 prior_question_elapsed_times_idxs_without_padding = user_data_seq[start_pos:start_pos+seq_len]
prior_question_elapsed_times_idxs_without_padding = np.power(np.minimum(prior_question_elapsed_times_idxs_without_padding // 1000, 300)+ 1, 0.726 ).astype(np.int8)
 prior_question_elapsed_times_idxs[:seq_len] = prior_question_elapsed_times_idxs_without_padding
bundle_elapsed_times_idxs = np.zeros(self.max_seq_len + 2, dtype=np.int8)
 bundle_elapsed_times_idxs[local_bundle_ids_without_padding - 1 + 1] = prior_question_elapsed_times_idxs_without_padding
bundle_elapsed_times_idxs = bundle_elapsed_times_idxs[1:]
question_elapsed_times_idxs_without_padding = bundle_elapsed_times_idxs[local_bundle_ids_without_padding]
post_question_elapsed_times_idxs = np.zeros(self.max_seq_len, dtype=np.int8)
 post_question_elapsed_times_idxs[:seq_len] = question_elapsed_times_idxs_without_padding
user_data_seq = user_data['prior_question_had_explanation_idxs']
prior_question_had_explanation_idxs = np.zeros(self.max_seq_len, dtype=np.int8)
 prior_question_had_explanation_idxs_without_padding = user_data_seq[start_pos:start_pos+seq_len]
prior_question_had_explanation_idxs[:seq_len] = prior_question_had_explanation_idxs_without_padding
bundle_has_explanation_idxs = np.zeros(self.max_seq_len + 2, dtype=np.int8)
 bundle_has_explanation_idxs[local_bundle_ids_without_padding - 1 + 1] = prior_question_had_explanation_idxs_without_padding
bundle_has_explanation_idxs = bundle_has_explanation_idxs[1:]
question_has_explanation_idxs_without_padding = bundle_has_explanation_idxs[local_bundle_ids_without_padding]
post_question_has_explanation_idxs = np.zeros(self.max_seq_len, dtype=np.int8)
 post_question_has_explanation_idxs[:seq_len] = question_has_explanation_idxs_without_padding
user_data_seq = user_data['user_answers']
user_answers_idxs = np.zeros(self.max_seq_len, dtype=np.int8)
 user_answers_idxs[:seq_len] = user_data_seq[start_pos:start_pos+seq_len] + 1
correct_answers_idxs = np.zeros(self.max_seq_len, dtype=np.int8)
 correct_answers_idxs[:seq_len] =(question_correct_answers_by_idxs + 1)[question_idxs_without_padding]
user_data_seq = user_data['answered_correctly_idxs']
answered_correctly_idxs = np.zeros(self.max_seq_len, dtype=user_data_seq.dtype)
 answered_correctly_idxs_without_padding = user_data_seq[start_pos:start_pos+seq_len]
answered_correctly_idxs[:seq_len] = answered_correctly_idxs_without_padding
prior_question_was_answered_correctly_idxs = np.zeros(self.max_seq_len, dtype=np.int8)
 prior_question_was_answered_correctly_idxs[1:seq_len] = answered_correctly_idxs_without_padding[:-1]
mask = np.zeros(self.max_seq_len, dtype=np.int8)
 mask[:seq_len] = 1
if self.is_train_set:
participates_in_roc_auc = mask
else:
participates_in_roc_auc = np.zeros(self.max_seq_len, dtype=np.int8)
 if set_roc_auc_start_poss - start_pos < seq_len - 1:
participates_in_roc_auc[set_roc_auc_start_poss - start_pos:seq_len - 1] = 1
return(last_bundle_mask,
question_timestamps,
question_pre_lecture_length_idxs,
question_idxs,
local_bundle_ids, prior_question_elapsed_times_idxs, prior_question_had_explanation_idxs,
question_from_prev_timestamps_idxs,
prior_question_was_answered_correctly_idxs,
post_question_elapsed_times_idxs, post_question_has_explanation_idxs,
user_answers_idxs,
correct_answers_idxs,
answered_correctly_idxs, participates_in_roc_auc, mask )<categorify>
SEED = [0]
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 test = test_short_form_loader(".. /input/lish-moa/test_features.csv")
 train = pd.concat([train,test])
 train[target_cols]= train[target_cols].fillna(0)
 test = train.copy()
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
train = multifold_indexer(train,target_cols,n_splits=NFOLDS)
 for fold in range(NFOLDS):
train_df = train[train['kfold'] != fold]
valid_df = train[train['kfold'] == fold]
feature_cols = [col for col in train_df.columns if not(col in target_cols+['kfold'])]
pipeline_val = make_pipeline(*transformers_list)
 X_train, y_train = train_df[feature_cols], train_df[target_cols]
X_valid, y_valid = valid_df[feature_cols], valid_df[target_cols].values
X_train = pipeline_val.fit_transform(X_train)
 X_train = X_train.values
X_valid = pipeline_val.transform(X_valid)
 valid_index = X_valid.index
X_valid = X_valid.values
y_train = y_train.values
X_test = test[feature_cols]
X_test = pipeline_val.transform(X_test ).values
pred_ = run_inference(X_train,y_train,X_valid,y_valid,X_test,fold, seed,inference_only=True)
 break
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(f'device: {device}')
 def _get_activation_fn(activation):
if activation == "relu":
return F.relu
elif activation == "gelu":
return F.gelu
raise RuntimeError("activation should be relu/gelu, not {}".format(activation))
use_reformer_self_attn = False
class TransformerEncoderLayer(nn.Module):
r
def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="relu"):
super(TransformerEncoderLayer, self ).__init__()
self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
 self.linear1 = nn.Linear(d_model, dim_feedforward)
 self.dropout = nn.Dropout(dropout)
 self.linear2 = nn.Linear(dim_feedforward, d_model)
 self.norm1 = nn.LayerNorm(d_model)
 self.norm2 = nn.LayerNorm(d_model)
 self.dropout1 = nn.Dropout(dropout)
 self.dropout2 = nn.Dropout(dropout)
 self.activation = _get_activation_fn(activation)
 def __setstate__(self, state):
if 'activation' not in state:
state['activation'] = F.relu
super(TransformerEncoderLayer, self ).__setstate__(state)
 def forward(self, src: torch.Tensor, src_mask: Optional[torch.Tensor] = None, src_key_padding_mask: Optional[torch.Tensor] = None)-> torch.Tensor:
r
if use_reformer_self_attn:
src2 = self.self_attn(src.transpose(0, 1 ).contiguous() 
)
 src = src + self.dropout1(src2.transpose(0, 1 ).contiguous())
 else:
src2 = self.self_attn(src, src, src, attn_mask=src_mask,
key_padding_mask=src_key_padding_mask)[0]
src = src + self.dropout1(src2)
 src = self.norm1(src)
 src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
src = src + self.dropout2(src2)
 src = self.norm2(src)
 return src
class TransformerEncoderOnlyModel(nn.Module):
def __init__(self, d_model, nhead, dim_feedforward, num_layers, max_seq_len, dropout=0.1, use_performer=True, feature_type='sqr', compute_type='iter', on_gptln=True):
super(TransformerEncoderOnlyModel, self ).__init__()
self.d_model = d_model
self.nhead = nhead
self.max_seq_len = max_seq_len
self.model_type = 'Transformer'
self.use_performer = use_performer
self.feature_type = feature_type
self.compute_type = compute_type
self.on_gptln = on_gptln
self.day_part_embedding = nn.Embedding(240, d_model)
 self.question_pre_lecture_length_idxs_embedding = nn.Embedding(32, d_model)
 question_idxs_cnt = len(question_ids_by_idxs)
 self.question_embedding = nn.Embedding(question_idxs_cnt, d_model)
 self.question_postfactumness_embedding = nn.Embedding(3, d_model)
 self.local_bundle_ids_embedding = nn.Embedding(max_seq_len + 1, d_model)
 self.prior_question_elapsed_times_idxs_embedding = nn.Embedding(64, d_model)
 self.prior_question_had_explanation_idxs_embedding = nn.Embedding(3, d_model)
 self.question_from_prev_timestamps_idxs_embedding = nn.Embedding(32, d_model)
 self.prior_question_was_answered_correctly_idxs_embedding = nn.Embedding(3, d_model)
 self.post_question_elapsed_times_idxs_embedding = nn.Embedding(64, d_model)
 self.post_question_has_explanation_idxs_embedding = nn.Embedding(3, d_model)
 self.question_was_answered_correctly_idxs_embedding = nn.Embedding(3, d_model)
 self.correct_answer_embedding = nn.Embedding(5, d_model)
 if self.use_performer:
vocab_size = None
self.transformer_encoder = slim_performer_model.SLiMPerformer(vocab_size, d_model,
num_layers, dim_feedforward,
nhead, feature_type,
compute_type,
on_gptln)
 else:
encoder_layer = TransformerEncoderLayer(d_model, nhead, dim_feedforward, dropout, activation='gelu')
 self.transformer_encoder = TransformerEncoder(encoder_layer, num_layers)
 self.encoder_top_linear_correct_answer_idx = nn.Linear(d_model, 1)
 self.encoder_top_linear_user_answer_idx = nn.Linear(d_model, 5)
 self.init_weights()
def generate_square_subsequent_mask(self, seq_len):
mask =(torch.triu(torch.ones(seq_len, seq_len)) == 1 ).transpose(0, 1)
 mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
return mask
def init_weights(self):
initrange = 0.00001
self.day_part_embedding.weight.data.uniform_(-initrange, initrange)
 self.question_pre_lecture_length_idxs_embedding.weight.data.uniform_(-initrange, initrange)
 self.question_embedding.weight.data.uniform_(-initrange, initrange)
 self.question_postfactumness_embedding.weight.data.uniform_(-initrange, initrange)
 self.local_bundle_ids_embedding.weight.data.uniform_(-initrange, initrange)
 self.prior_question_elapsed_times_idxs_embedding.weight.data.uniform_(-initrange, initrange)
 self.prior_question_had_explanation_idxs_embedding.weight.data.uniform_(-initrange, initrange)
 self.question_from_prev_timestamps_idxs_embedding.weight.data.uniform_(-initrange, initrange)
 self.prior_question_was_answered_correctly_idxs_embedding.weight.data.uniform_(-initrange, initrange)
 self.post_question_elapsed_times_idxs_embedding.weight.data.uniform_(-initrange, initrange)
 self.post_question_has_explanation_idxs_embedding.weight.data.uniform_(-initrange, initrange)
 self.question_was_answered_correctly_idxs_embedding.weight.data.uniform_(-initrange, initrange)
 self.correct_answer_embedding.weight.data.uniform_(-initrange, initrange)
 def forward(self,
user_idxs,
timestamps,
question_pre_lecture_length_idxs,
question_idxs,
local_bundle_ids,
prior_question_elapsed_times_idxs,
prior_question_had_explanation_idxs,
question_from_prev_timestamps_idxs,
prior_question_was_answered_correctly_idxs,
post_question_elapsed_times_idxs,
post_question_has_explanation_idxs,
question_was_answered_correctly_idxs,
correct_answers_idxs,
mask,
attn_mask):
day_part_embedding = self.day_part_embedding(((timestamps % 86400000)*(240.0 / 86400000)).long())
 question_pre_lecture_length_idxs_embedding = self.question_pre_lecture_length_idxs_embedding(question_pre_lecture_length_idxs)
 question_embedding = self.question_embedding(question_idxs)
 question_postfactumness_embedding_x2 = self.question_postfactumness_embedding(mask.unsqueeze(-1)* torch.tensor([1, 2] ).to(device))
local_bundle_ids_embedding = self.local_bundle_ids_embedding(local_bundle_ids)
 prior_question_elapsed_times_idxs_embedding = self.prior_question_elapsed_times_idxs_embedding(prior_question_elapsed_times_idxs)
 prior_question_had_explanation_idxs_embedding = self.prior_question_had_explanation_idxs_embedding(prior_question_had_explanation_idxs)
 question_from_prev_timestamps_idxs_embedding = self.question_from_prev_timestamps_idxs_embedding(question_from_prev_timestamps_idxs)
 prior_question_was_answered_correctly_idxs_embedding = self.prior_question_was_answered_correctly_idxs_embedding(prior_question_was_answered_correctly_idxs)
 post_question_elapsed_times_idxs_embedding = self.post_question_elapsed_times_idxs_embedding(post_question_elapsed_times_idxs)
 post_question_has_explanation_idxs_embedding = self.post_question_has_explanation_idxs_embedding(post_question_has_explanation_idxs)
 question_was_answered_correctly_idxs_embedding = self.question_was_answered_correctly_idxs_embedding(question_was_answered_correctly_idxs)
 correct_answer_embedding = self.correct_answer_embedding(correct_answers_idxs)
 question_related_embeddings =(
 question_pre_lecture_length_idxs_embedding
+ question_embedding
+ day_part_embedding
+ local_bundle_ids_embedding
+ question_from_prev_timestamps_idxs_embedding
+ correct_answer_embedding 
)
 question_related_embeddings_x2 = question_related_embeddings.unsqueeze(-2)* torch.tensor([[1.], [1.]] ).to(device)
 posts_x2 =(post_question_elapsed_times_idxs_embedding + post_question_has_explanation_idxs_embedding + question_was_answered_correctly_idxs_embedding ).unsqueeze(-2)* torch.tensor([[0.], [1.]] ).to(device)
 encoder_input_x2 = question_postfactumness_embedding_x2 + question_related_embeddings_x2 + posts_x2
encoder_input = encoder_input_x2.view([encoder_input_x2.size() [0], encoder_input_x2.size() [1] * encoder_input_x2.size() [2], encoder_input_x2.size() [3]])
 if not self.use_performer:
encoder_input = encoder_input.transpose(0, 1)
 output = self.transformer_encoder(encoder_input, attn_mask)
 else:
output = self.transformer_encoder.full_forward(encoder_input)
 output0 = self.encoder_top_linear_correct_answer_idx(output)
 if not self.use_performer:
output0 = output0.view([output0.size() [0] // 2, 2, output0.size() [1], output0.size() [2]])[:,0,:,:]
output0 = output0.transpose(0, 1 ).contiguous()
else:
output0 = output0.view([output0.size() [0], output0.size() [1] // 2, 2, output0.size() [2]])[:,:,0,:]
output1 = self.encoder_top_linear_user_answer_idx(output)
 if not self.use_performer:
output1 = output1.view([output1.size() [0] // 2, 2, output1.size() [1], output1.size() [2]])[:,0,:,:]
output1 = output1.transpose(0, 1 ).contiguous()
else:
output1 = output1.view([output1.size() [0], output1.size() [1] // 2, 2, output1.size() [2]])[:,:,0,:]
return output0, output1<categorify>
transformed_features = pd.DataFrame(pred_[1],index=test.index )
class CustomBCELoss(nn.Module):
def __init__(self):
super().__init__()
def forward(self, output, target, **kwargs):
loss =(( target - 1)* torch.log(output)+(2 - target)* torch.log(1.0 - output)) *(target != 0 ).float()
loss = torch.neg(torch.mean(loss))
return loss
WARMUP_EPOCHS = 1.0 / 4.0
epochs = 5
lr = 0.0005
dropout = 0.0
MAX_SEQ_LEN = 256
batch_size = 64
valid_batch_size = 384
model = TransformerEncoderOnlyModel(d_model=512, nhead=8, dim_feedforward=384, num_layers=6, max_seq_len=MAX_SEQ_LEN*2, dropout=dropout, use_performer=False, feature_type='relu', compute_type='iter', on_gptln=True)
 criteria = [
CustomBCELoss().to(device),
nn.CrossEntropyLoss(ignore_index=0, weight=torch.tensor([0, 0.25, 0.25, 0.25, 0.25,])).to(device),
]
best_model_file_path = '.. /input/model4pth/prf_wue0.25_sl256_lr0.0005_e5_of7_trainauc0.8082896856958341.pth'
print('Loading Model:', best_model_file_path)
 model = torch.load(best_model_file_path)
 model.to(device)
 test_set_start_positions_by_user_idxs, test_user_idxs, test_start_poss, test_lengths, test_roc_auc_start_poss = sample_test(MAX_SEQ_LEN)
 valid_dataset = RiiidDataset(is_train_set=False, max_seq_len=MAX_SEQ_LEN)
 print('MAX_SEQ_LEN:', MAX_SEQ_LEN, model.max_seq_len)
 print('epochs:', epochs)
 print('WARMUP_EPOCHS:', WARMUP_EPOCHS)
 print('lr:', lr)
 print('dropout:', dropout)
 print('batch_size:', batch_size)
 print('valid_batch_size:', valid_batch_size)
 print('nhead:', model.nhead)
 print(model)
 gc.collect()<load_from_csv>
transformed_features.columns = [str(i)for i in range(len(transformed_features.columns)) ]
<feature_engineering>
corr_matrix = transformed_features.corr().abs()
sol =(corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool)) 
.stack() 
.sort_values(ascending=False))
class Iter_Valid(object):
def __init__(self, df, max_user=1000):
df = df.reset_index(drop=True)
 self.df = df
self.user_answer = df['user_answer'].astype(str ).values
self.answered_correctly = df['answered_correctly'].astype(str ).values
df['prior_group_responses'] = "[]"
df['prior_group_answers_correct'] = "[]"
self.sample_df = df[df['content_type_id'] == 0][['row_id']]
self.sample_df['answered_correctly'] = 0
self.len = len(df)
 self.user_id = df.user_id.values
self.task_container_id = df.task_container_id.values
self.content_type_id = df.content_type_id.values
self.max_user = max_user
self.current = 0
self.pre_user_answer_list = []
self.pre_answered_correctly_list = []
def __iter__(self):
return self
def fix_df(self, user_answer_list, answered_correctly_list, pre_start):
df= self.df[pre_start:self.current].copy()
sample_df = self.sample_df[pre_start:self.current].copy()
df.loc[pre_start,'prior_group_responses'] = '[' + ",".join(self.pre_user_answer_list)+ ']'
df.loc[pre_start,'prior_group_answers_correct'] = '[' + ",".join(self.pre_answered_correctly_list)+ ']'
print('lll',df.loc[pre_start,'prior_group_answers_correct'])
 self.pre_user_answer_list = user_answer_list
self.pre_answered_correctly_list = answered_correctly_list
return df, sample_df
def __next__(self):
found_good = False
while not found_good:
added_user = set()
pre_start = self.current
pre_added_user = -1
pre_task_container_id = -1
user_answer_list = []
answered_correctly_list = []
result = None
while self.current < self.len:
crr_user_id = self.user_id[self.current]
crr_task_container_id = self.task_container_id[self.current]
crr_content_type_id = self.content_type_id[self.current]
if crr_content_type_id == 1:
self.current += 1
continue
if crr_user_id in added_user and(( crr_user_id != pre_added_user)or 
(crr_task_container_id != pre_task_container_id)) :
result = self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 break
if len(added_user)== self.max_user:
if crr_user_id == pre_added_user and crr_task_container_id == pre_task_container_id:
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
continue
else:
result = self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 break
added_user.add(crr_user_id)
 pre_added_user = crr_user_id
pre_task_container_id = crr_task_container_id
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
if result is None:
if pre_start < self.current:
result = self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 else:
raise StopIteration()
dff = result[0]
if len(dff[dff['content_type_id'] == 0])> 0:
found_good = True
return dff, result[1]
else:
found_good = False 
<split>
sum(sol > 0.5 )
env = riiideducation.make_env()
iter_test = env.iter_test()<define_variables>
multin = sol.loc[sol > 0.5].index
user_data_by_idxs = list(user_data_by_idxs)
 user_ids_by_idxs = list(user_ids_by_idxs)
 fake_dataset = RiiidDataset(is_train_set=False, max_seq_len=MAX_SEQ_LEN)
 model.eval()
def process_batch(batch_as_item, batch_positions_of_interest, batch_id):
item = batch_as_item
losses = []
losses_sum = 0.0
num_corrects = 0
num_total = 0
labels = []
outputs = []
predictions = []
with torch.no_grad() :
attn_mask = model.generate_square_subsequent_mask(seq_len=model.max_seq_len ).to(device)
 user_idxs = item[0].to(device ).long()
timestamps = item[1].to(device ).double()
question_pre_lecture_length_idxs = item[2].to(device ).long()
question_idxs = item[3].to(device ).long()
local_bundle_ids = item[4].to(device ).long()
prior_question_elapsed_times_idxs = item[5].to(device ).long()
prior_question_had_explanation_idxs = item[6].to(device ).long()
question_from_prev_timestamps_idxs = item[7].to(device ).long()
prior_question_was_answered_correctly_idxs = item[8].to(device ).long()
post_question_elapsed_times_idxs = item[9].to(device ).long()
post_question_has_explanation_idxs = item[10].to(device ).long()
label_user_answers_idxs = item[11].to(device ).long()
correct_answers_idxs = item[12].to(device ).long()
label_answered_correctly_idxs = item[-3].to(device ).long()
participates_in_roc_auc_long = item[-2]
participates_in_roc_auc_long = participates_in_roc_auc_long.to(device ).long()
participates_in_roc_auc = participates_in_roc_auc_long.float()
mask = item[-1].to(device ).long()
output0, output1 = model(user_idxs,
timestamps,
question_pre_lecture_length_idxs,
question_idxs,
local_bundle_ids, prior_question_elapsed_times_idxs, prior_question_had_explanation_idxs,
question_from_prev_timestamps_idxs,
prior_question_was_answered_correctly_idxs,
post_question_elapsed_times_idxs,
post_question_has_explanation_idxs,
label_answered_correctly_idxs,
correct_answers_idxs,
mask,
attn_mask)
 output0 = output0.squeeze(-1)
 output0 = torch.sigmoid(output0)
 pred =(output0 >= 0.5 ).long()
pred = pred + 1
idx0 = torch.arange(len(batch_positions_of_interest)).to(device)
 idx1 = torch.tensor(batch_positions_of_interest ).to(device)
 batch_predictions = output0[idx0, idx1]
batch_predictions = batch_predictions.detach()
batch_predictions = batch_predictions.cpu()
batch_predictions = batch_predictions.numpy().astype(np.float64)
 loss0 = criteria[0](output0.view(-1), label_answered_correctly_idxs.float().view(-1))
loss1 = criteria[1](output1.reshape(-1, 5), label_user_answers_idxs.view(-1))
loss =(loss0 + loss1)
 loss_val = loss.detach().item()
losses.append(loss_val)
 losses_sum += loss_val
num_corrects +=(( pred == label_answered_correctly_idxs)* participates_in_roc_auc ).sum().item()
num_total += participates_in_roc_auc.sum().item()
auc_roc_applicable_positions_mask_flattened =(participates_in_roc_auc * label_answered_correctly_idxs ).bool().view(-1 ).cpu().numpy()
label_answered_correctly_idxs_flattened =(label_answered_correctly_idxs - 1 ).view(-1 ).cpu().numpy()
labels.extend(label_answered_correctly_idxs_flattened[auc_roc_applicable_positions_mask_flattened])
 output_flattened = output0.view(-1 ).data.cpu().numpy()
outputs.extend(output_flattened[auc_roc_applicable_positions_mask_flattened])
 if batch_id % 100 == 0:
if len(outputs)> 0:
acc = num_corrects / max(0.000001, num_total)
 if np.max(labels)!= np.min(labels):
auc = roc_auc_score(labels, outputs)
 else:
auc = 0.0
loss = np.mean(losses)
 else:
acc = 0.0
auc = 0.0
loss = np.mean(losses)
 print(f'{datetime.now().isoformat() }: batch_id={batch_id} batch_len={len(batch)} loss={loss}, acc={acc}, auc={auc}')
 return batch_predictions
def update_prev_group(prior_group_answers_correct, prior_group_responses, prior_group_user_idxs):
prior_group_answers_correct_idxs = np.array([int(s)+ 1 for s in(prior_group_answers_correct[1:-1].replace(' ', '' ).split(',')if prior_group_answers_correct != '[]' else [])], dtype=np.int8)
 prior_group_responses = np.array([int(s)for s in(prior_group_responses[1:-1].replace(' ', '' ).split(',')if prior_group_responses != '[]' else [])], dtype=np.int8)
 assert len(prior_group_answers_correct_idxs)== len(prior_group_responses)
 assert len(prior_group_answers_correct_idxs)== len(prior_group_user_idxs), f'len(prior_group_answers_correct_idxs)== len(prior_group_user_idxs): {len(prior_group_answers_correct_idxs)} {len(prior_group_user_idxs)}'
user_data_skips_by_idxs = np.zeros(len(user_data_by_idxs), dtype='int')
 for i in range(len(prior_group_responses)- 1, -1, -1):
user_idx = prior_group_user_idxs[i]
if user_idx != 0:
skip = user_data_skips_by_idxs[user_idx]
user_data_skips_by_idxs[user_idx] += 1
user_data = user_data_by_idxs[user_idx]
user_answers = user_data['user_answers']
assert len(user_answers)> skip, f'len(user_answers)>= skip: {len(user_answers)} > {skip}'
user_answers[-skip-1] = prior_group_responses[i]
answered_correctly_idxs = user_data['answered_correctly_idxs']
assert len(answered_correctly_idxs)> skip, f'len(answered_correctly_idxs)> skip: {len(answered_correctly_idxs)} >= {skip}'
answered_correctly_idxs[-skip-1] = prior_group_answers_correct_idxs[i]
group_user_idxs = []
batch_id = 0
print(f'{datetime.now().isoformat() }: Starting dfs')
 for(test_df, sample_prediction_df)in iter_test:
dbg_cond = batch_id < 10
if dbg_cond:
print(f'{datetime.now().isoformat() }: Starting df of size {len(test_df)}({len(test_df[test_df["content_type_id"] == 0])}q {len(test_df[test_df["content_type_id"] == 1])}l)sample_prediction_df: {sample_prediction_df}')
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].astype(np.float ).fillna(-1 ).astype(np.int8)
 test_df['prior_question_elapsed_time'] = test_df['prior_question_elapsed_time'].fillna(-1000)
 if dbg_cond:
print(test_df.head(10))
answered_correctly = [0.0 for l in range(len(test_df)) ]
prior_group_user_idxs = group_user_idxs
group_user_idxs = []
prev_group_num = -1
batch_samples = [[] for i in range(16)]
batch_positions_to_df_positions = []
batch_positions_of_interest = []
for df_position,(group_num, row)in enumerate(test_df.iterrows()):
timestamp = row['timestamp']
user_id = row['user_id']
content_id = row['content_id']
content_type_id = row['content_type_id']
task_container_id = row['task_container_id']
prior_question_elapsed_time = row['prior_question_elapsed_time']
prior_question_had_explanation = row['prior_question_had_explanation']
prior_group_answers_correct = row['prior_group_answers_correct']
prior_group_responses = row['prior_group_responses']
assert content_type_id == 0 or content_type_id == 1
if dbg_cond:
print('row, group_num:', row, group_num)
 if df_position == 0:
update_prev_group(prior_group_answers_correct, prior_group_responses, prior_group_user_idxs)
 if user_id in user_idxs_by_ids:
user_idx = user_idxs_by_ids[user_id]
user_data = user_data_by_idxs[user_idx]
else:
user_data = {
'question_timestamps': np.array([], dtype=np.int64),
'question_prev_timestamps': np.array([], dtype=np.int64),
'question_ids': np.array([], dtype=np.int16),
'question_task_container_ids': np.array([], dtype=np.int16),
'user_answers': np.array([], dtype=np.int8),
'prior_question_elapsed_times': np.array([], dtype=np.float32),
'prior_question_had_explanation_idxs': np.array([], dtype=np.int8),
'answered_correctly_idxs': np.array([], dtype=np.int8),
'question_prev_was_lecture': np.array([], dtype=np.int8),
'question_prev_prev_timestamps': np.array([], dtype=np.int64),
}
user_idx = len(user_data_by_idxs)
 user_data_by_idxs.append(user_data)
 user_idxs_by_ids[user_id] = user_idx
user_data_questions_counts_by_idxs.append(0)
 user_ids_by_idxs.append(user_id)
 if content_type_id == 1:
user_data['reminder__question_prev_lecture_timestamp'] =(timestamp)
 group_user_idxs.append(0)
 else:
if 'reminder__question_prev_lecture_timestamp' in user_data:
reminder__question_prev_lecture_timestamp = user_data['reminder__question_prev_lecture_timestamp']
reminder__prev_was_lecture = 1
del user_data['reminder__question_prev_lecture_timestamp']
else:
reminder__question_prev_lecture_timestamp = None
reminder__prev_was_lecture = 0
user_data_questions_counts_by_idxs[user_idx] += 1
question_timestamps = user_data['question_timestamps']
question_timestamps = np.append(question_timestamps, timestamp)
 if len(question_timestamps)> MAX_SEQ_LEN + 2:
question_timestamps = question_timestamps[-MAX_SEQ_LEN-2:]
user_data_questions_counts_by_idxs[user_idx] = MAX_SEQ_LEN + 2
user_data['question_timestamps'] = question_timestamps
question_prev_timestamps = user_data['question_prev_timestamps']
if reminder__question_prev_lecture_timestamp is not None:
question_prev_timestamp = reminder__question_prev_lecture_timestamp
else:
if len(question_timestamps)>= 2:
question_prev_timestamp = question_timestamps[-2]
else:
question_prev_timestamp = 0
question_prev_timestamps = np.append(question_prev_timestamps, question_prev_timestamp)
 if len(question_prev_timestamps)> MAX_SEQ_LEN + 2:
question_prev_timestamps = question_prev_timestamps[-MAX_SEQ_LEN-2:]
user_data['question_prev_timestamps'] = question_prev_timestamps
question_ids = user_data['question_ids']
question_ids = np.append(question_ids, content_id)
 if len(question_ids)> MAX_SEQ_LEN + 2:
question_ids = question_ids[-MAX_SEQ_LEN-2:]
user_data['question_ids'] = question_ids
question_task_container_ids = user_data['question_task_container_ids']
question_task_container_ids = np.append(question_task_container_ids, task_container_id)
 if len(question_task_container_ids)> MAX_SEQ_LEN + 2:
question_task_container_ids = question_task_container_ids[-MAX_SEQ_LEN-2:]
user_data['question_task_container_ids'] = question_task_container_ids
user_answers = user_data['user_answers']
user_answers = np.append(user_answers, 0)
 if len(user_answers)> MAX_SEQ_LEN + 2:
user_answers = user_answers[-MAX_SEQ_LEN-2:]
user_data['user_answers'] = user_answers
prior_question_elapsed_times = user_data['prior_question_elapsed_times']
prior_question_elapsed_times = np.append(prior_question_elapsed_times, prior_question_elapsed_time)
 if len(prior_question_elapsed_times)> MAX_SEQ_LEN + 2:
prior_question_elapsed_times = prior_question_elapsed_times[-MAX_SEQ_LEN-2:]
user_data['prior_question_elapsed_times'] = prior_question_elapsed_times
prior_question_had_explanation_idxs = user_data['prior_question_had_explanation_idxs']
prior_question_had_explanation_idxs = np.append(prior_question_had_explanation_idxs, prior_question_had_explanation + 1)
 if len(prior_question_had_explanation_idxs)> MAX_SEQ_LEN + 2:
prior_question_had_explanation_idxs = prior_question_had_explanation_idxs[-MAX_SEQ_LEN-2:]
user_data['prior_question_had_explanation_idxs'] = prior_question_had_explanation_idxs
answered_correctly_idxs = user_data['answered_correctly_idxs']
answered_correctly_idxs = np.append(answered_correctly_idxs, 0)
 if len(answered_correctly_idxs)> MAX_SEQ_LEN + 2:
answered_correctly_idxs = answered_correctly_idxs[-MAX_SEQ_LEN-2:]
user_data['answered_correctly_idxs'] = answered_correctly_idxs
question_prev_was_lecture = user_data['question_prev_was_lecture']
question_prev_was_lecture = np.append(question_prev_was_lecture, reminder__prev_was_lecture)
 if len(question_prev_was_lecture)> MAX_SEQ_LEN + 2:
question_prev_was_lecture = question_prev_was_lecture[-MAX_SEQ_LEN-2:]
user_data['question_prev_was_lecture'] = question_prev_was_lecture
question_prev_prev_timestamps = user_data['question_prev_prev_timestamps']
if len(question_prev_timestamps)>= 2:
question_prev_prev_timestamp = question_prev_timestamps[-2]
else:
question_prev_prev_timestamp = 0
question_prev_prev_timestamps = np.append(question_prev_prev_timestamps, question_prev_prev_timestamp)
 if len(question_prev_prev_timestamps)> MAX_SEQ_LEN + 2:
question_prev_prev_timestamps = question_prev_prev_timestamps[-MAX_SEQ_LEN-2:]
user_data['question_prev_prev_timestamps'] = question_prev_prev_timestamps
if dbg_cond:
print('u_idx:', user_idx, batch_id)
 group_user_idxs.append(user_idx)
 sample = fake_dataset.__getitem__(-user_idx)
 mask = sample[-1]
positions_of_interest = np.nonzero(mask)[0]
if len(positions_of_interest)> 0:
position_of_interest = positions_of_interest[-1]
else:
position_of_interest = 0
batch_positions_to_df_positions.append(df_position)
 batch_positions_of_interest.append(position_of_interest)
 for i in range(len(sample)) :
batch_samples[i].append(sample[i])
 need_to_update_batch_id = False
if(len(batch_samples[0])== valid_batch_size)or(df_position == len(test_df)- 1):
if len(batch_samples[0])> 0:
batch = [torch.tensor(batch_samples[i])for i in range(len(batch_samples)) ]
if dbg_cond:
print(f'{datetime.now().isoformat() }: Predicting batch {batch_id}')
 for ii in range(16):
print(batch_samples[ii])
 batch_predictions = process_batch(batch, batch_positions_of_interest, batch_id)
 for batch_pos, batch_position_to_df_position in enumerate(batch_positions_to_df_positions):
answered_correctly[batch_position_to_df_position] = batch_predictions[batch_pos]
batch_samples = [[] for i in range(16)]
batch_positions_to_df_positions = []
batch_positions_of_interest = []
need_to_update_batch_id = True
if df_position == len(test_df)- 1:
assert len(test_df)== len(answered_correctly)
 if dbg_cond:
print(f'{datetime.now().isoformat() }: Sending df predictions')
 test_df['answered_correctly'] = np.array(answered_correctly)
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])
 if dbg_cond:
print(f'{datetime.now().isoformat() }: Sent df predictions')
 if need_to_update_batch_id:
batch_id += 1
if dbg_cond:
print(f'{datetime.now().isoformat() }: Finished df')
 print(f'{datetime.now().isoformat() }: Done' )<install_modules>
len(set(multin.get_level_values(0 ).unique().tolist() + multin.get_level_values(1 ).unique().tolist()))
try:
except ModuleNotFoundError:
!pip install --use-feature=2020-resolver zarr > /dev/null
try:
except ModuleNotFoundError:
!pip install --use-feature=2020-resolver pytorch-lightning==0.8.4 > /dev/null<import_modules>
from sklearn.base import BaseEstimator, TransformerMixin
import models.pointnet as pointnet
from training.trainer import get_last_checkpoint, BaseLightningModule
from training.configs.base import GenericConfig
from data.dataset import collate_V5 as collate, CustomLyftDataset_V7 as CustomLyftDataset<import_modules>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
from pathlib import Path
import zarr
from tqdm.notebook import tqdm
import numpy as np, pandas as pd
import torch, torch.nn as nn
from torch.utils.data import DataLoader<import_modules>
def seed_everything(seed=42):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
import bisect, yaml
import itertools as it<choose_model_class>
class DaeAdder(BaseEstimator, TransformerMixin):
def __init__(self,filename):
self.filename=filename
def fit(self,X,y=None):
return self
def fit_transform(self,X,y=None):
Dae_features = pd.read_feather(self.filename ).set_index('sig_id')
 return X.merge(Dae_features,how='left', on='sig_id')
 def transform(self,X):
Dae_features = pd.read_feather(self.filename ).set_index('sig_id')
 return X.merge(Dae_features,how='left', on='sig_id')
pool = Parallel(3 )<define_variables>
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv' )
DATA_ROOT = Path(r".. /input/lyft-motion-prediction-autonomous-vehicles")
 ZARR_PATH = Path("scenes/test.zarr")
 HPARAM_PATH = r".. /input/best-submission/hparams.yaml"
print("DATA_ROOT: {}
ZARR_PATH: {}".format(DATA_ROOT, ZARR_PATH))<load_pretrained>
Dae0_2 =DaeAdder(filename='features_0.2_altogether.fth')
%%time
mask = np.load(DATA_ROOT/"scenes/mask.npz")['arr_0']
mask<define_variables>
res = Dae0_2.fit_transform(train )
agent_ids = np.where(mask)[0]
agent_ids<load_pretrained>
class SupressControls(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
X = X[X['cp_type']!=0]
X = X.drop('cp_type', axis=1)
 return X
def transform(self, X):
X = X[X['cp_type']!=0]
X = X.drop('cp_type', axis=1)
 return X
class CatIntMapper(BaseEstimator, TransformerMixin):
def __init__(self ,col,dicti):
self.col = col
self.dicti = dicti
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
assert X[self.col].isin(self.dicti.keys() ).all()
return pd.concat([X.drop(self.col,axis=1),X[self.col].map(self.dicti ).astype(int ).rename(self.col)],axis=1)
 def transform(self, X):
assert X[self.col].isin(self.dicti.keys() ).all()
return pd.concat([X.drop(self.col,axis=1),X[self.col].map(self.dicti ).astype(int ).rename(self.col)],axis=1)
 class NamedOutTWrapper(BaseEstimator, TransformerMixin):
def __init__(self,transformer,columns,inplace=False,prefix='_'):
self.transformer = transformer
self.cols = columns
self.inplace = inplace
self.prefix = prefix
self.transformer_name = self._get_transformer_name()
def fit(self, X, y = None):
self.transformer = self.transformer.fit(X[self.cols] , y)
 return self
def fit_transform(self, X, y = None):
transformed_columns = self.transformer.fit_transform(X[self.cols] , y)
 out=pd.DataFrame(index=X.index)
 if self.inplace:
out = X[self.cols].copy()
out[self.cols] = transformed_columns
return pd.concat([X.drop(self.cols,axis=1),out],axis=1)
 else:
for i,values in enumerate(transformed_columns.transpose()):
out[ self.transformer_name + self.prefix + str(i)] = values
return pd.concat([X,out],axis=1)
 def transform(self, X):
transformed_columns = self.transformer.transform(X[self.cols])
 out=pd.DataFrame(index=X.index)
 if self.inplace:
out = X[self.cols].copy()
out[self.cols] = transformed_columns
return pd.concat([X.drop(self.cols,axis=1),out],axis=1)
 else:
for i,values in enumerate(transformed_columns.transpose()):
out[ self.transformer_name + self.prefix + str(i)] = values
return pd.concat([X,out],axis=1)
 def _get_transformer_name(self):
return str(self.transformer.__class__ ).split('.')[-1][0:-2]
class IdentityTransformer:

 def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X
def transform(self, X):
return X
class SuppressControls(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X.loc[X['cp_type']=='trt_cp'].drop('cp_type', axis=1)
 def transform(self, X):
return X.loc[X['cp_type']=='trt_cp'].drop('cp_type', axis=1)
 class ColumnDropper(BaseEstimator, TransformerMixin):
def __init__(self, cols):
self.cols=cols
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
return X.drop(self.cols,axis=1)
 class VarianceFilter(BaseEstimator, TransformerMixin):
def __init__(self,threshold):
self.threshold = threshold
def fit(self,X,y=None):
mask = X.var() <= self.threshold
self.drop_cols = set([ col for val,col in zip(mask,X.columns)if val])
 self.drop_cols.discard('cp_type')
 return self
def transform(self,X):
return X.drop(self.drop_cols,axis=1)
 def apply_pipe_together(pipeline,train,test):
data = pd.concat([train,test])
 data = pipeline.fit_transform(data)
 train = data.loc[data.index.intersection(train.index)]
test = data.loc[data.index.intersection(test.index)]
return pipeline,train,test
with open(HPARAM_PATH)as f:
hparams = yaml.load(f, Loader=yaml.SafeLoader)
 hparams<define_variables>
map_controls = CatIntMapper('cp_type',{'ctl_vehicle': 0, 'trt_cp': 1})
 map_dose = CatIntMapper('cp_dose',{'D1': 1, 'D2': 0})
 map_time = CatIntMapper('cp_time',{24: 0, 48: 1, 72: 2})
HBACKWARD = hparams["data_options"]["train"]["kwargs"]["hbackward"]
FRAME_STRIDE = hparams["data_options"]["train"]["kwargs"]["frame_stride"]
AGENT_FEATURE_DIM = hparams["data_options"]["train"]["kwargs"]["agent_feature_dim"]
MAX_AGENTS = hparams["data_options"]["train"]["kwargs"]["max_agents"]
HBACKWARD,FRAME_STRIDE, AGENT_FEATURE_DIM, MAX_AGENTS<find_best_params>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
hparams["model_options"]["kwargs"]<define_variables>
def groupkfold_indexer(train,target_columns,n_splits=10,random_state=12347,**kwargs):
folds = train.copy()
groups =pd.read_csv('.. /input/inferreddrugexperimentclustermoa/aggclus4_results.csv',index_col='sig_id')['aggclus1-clusters']
gkf = GroupKFold(n_splits=n_splits)
 folds[ 'kfold']=0
for f,(t_idx, v_idx)in enumerate(gkf.split(X=train.sample(len(train),random_state=random_state), y=train[target_columns].sample(len(train),random_state=random_state),groups= groups.loc[train.index].sample(len(train),random_state=random_state))):
folds.iloc[v_idx,-1] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 return folds
HFORWARD = 0
NFRAMES = 1<load_pretrained>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size1=388,hidden_size2=512,drop_rate1=0.0,drop_rate2=0.3,drop_rate3=0.3):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(drop_rate1)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size1))
self.batch_norm2 = nn.BatchNorm1d(hidden_size1)
 self.dropout2 = nn.Dropout(drop_rate2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size1, hidden_size2))
self.batch_norm3 = nn.BatchNorm1d(hidden_size2)
 self.dropout3 = nn.Dropout(drop_rate3)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size2, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
z = zarr.open(DATA_ROOT.joinpath(ZARR_PATH ).as_posix() )<feature_engineering>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
if not scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, scheduler, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 if scheduler.__class__ == torch.optim.lr_scheduler.ReduceLROnPlateau:
scheduler.step(final_loss)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
agents_ij = z.frames["agent_index_interval"]<import_modules>
def run_training(X_train,y_train,X_valid,y_valid,X_test,fold, seed,verbose=False,**kwargs):
seed_everything(seed)
 train_dataset = MoADataset(X_train, y_train)
 valid_dataset = MoADataset(X_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features= X_train.shape[1] ,
num_targets= y_train.shape[1],hidden_size1=hidden_size1,hidden_size2=hidden_size2,
**kwargs 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,patience=3)
 loss_val = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 if verbose:
print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model,scheduler, loss_val, validloader, DEVICE)
 if verbose:
print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof = valid_preds
elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
torch.save(model.state_dict() , f"{exp_name}_SEED{seed}_FOLD{fold}.pth")
 testdataset = TestDataset(X_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
<create_dataframe>
DEVICE = torch.device('cuda:0')
 EPOCHS = 70
BATCH_SIZE = 1024
LEARNING_RATE = 6e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
hidden_size1=2048
hidden_size2=2048
dt = CustomLyftDataset(
 z,
nframes=NFRAMES,
frame_stride=FRAME_STRIDE,
hbackward=HBACKWARD,
hforward=HFORWARD,
max_agents=MAX_AGENTS,
agent_feature_dim=AGENT_FEATURE_DIM, 
)
 dt.nread_frames<set_options>
transformers_list=[map_dose,map_time,Dae0_2,SuppressControls() ]
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu" )<find_best_params>
def run_k_fold(folds,target_cols,test,transformers_list,NFOLDS, seed,verbose=False,**kwargs):
train = folds
test_ = test
oof = train[target_cols].copy()
oof = oof*0
predictions = pd.DataFrame(0,columns=target_cols,index=test.index)
 for fold in range(NFOLDS):
train_df = train[train['kfold'] != fold]
valid_df = train[train['kfold'] == fold]
feature_cols = [col for col in train_df.columns if not(col in target_cols+['kfold'])]
pipeline_val = make_pipeline(*transformers_list)
 X_train, y_train = train_df[feature_cols], train_df[target_cols]
X_valid, y_valid = valid_df[feature_cols], valid_df[target_cols].values
X_train = pipeline_val.fit_transform(X_train,y_train)
 feature_cols = [col for col in X_train.columns if not(col in target_cols+['kfold'])]
X_train = X_train.values
X_valid = pipeline_val.transform(X_valid)
 valid_index = X_valid.index
X_valid = X_valid.values
y_train = y_train.values
X_test = pipeline_val.transform(test_)
 test_index = X_test.index
X_test = X_test[feature_cols].values
oof_, pred_ = run_training(X_train,y_train,X_valid,y_valid,X_test,fold, seed,verbose,**kwargs)
 oof.loc[valid_index] = oof_
predictions.loc[test_index] += pred_ / NFOLDS
return oof, predictions
model_name = "ListStackedPointNetWithProjection_V3"
model_options = GenericConfig(
 "model",
instance_name=model_name,
kwargs=hparams["model_options"]["kwargs"],
context={"model": getattr(pointnet, model_name)}, 
)
 model_options<load_pretrained>
params={}
def load_model(path):
model = BaseLightningModule(model_options ).to(DEVICE)
 print(model.load_state_dict(torch.load(path, map_location=DEVICE)["state_dict"]))
model.eval()
return model<define_variables>
SEED = [0]
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 test = test_short_form_loader(".. /input/lish-moa/test_features.csv")
 pipeline_test = make_pipeline(*transformers_list)
 pipeline_test,train , test = apply_pipe_together(pipeline_test,train,test)
 transformers_list=[IdentityTransformer() ]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
folds = groupkfold_indexer(train,target_cols,n_splits=NFOLDS)
 oof_, predictions_ = run_k_fold(folds,target_cols,test,transformers_list,NFOLDS, seed,verbose=True,**params)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 test[target_cols] = predictions
paths = [
".. /input/best-submission/model_epoch15.ckpt",
".. /input/best-submission/model_epoch45.ckpt",
".. /input/best-submission/model_epoch46.ckpt",
".. /input/best-submission/model_epoch47.ckpt",
".. /input/best-submission/model_epoch50.ckpt",
".. /input/best-submission/model_epoch51.ckpt",
".. /input/best-submission/model_epoch52.ckpt",
".. /input/best-submission/model_epoch53.ckpt",
".. /input/best-submission/model_epoch54.ckpt",
".. /input/best-submission/model_epoch55(1 ).ckpt",
".. /input/best-submission/model_epoch56.ckpt",
".. /input/best-submission/model_epoch57.ckpt",
".. /input/best-submission/model_epoch58.ckpt",
".. /input/best-submission/model_epoch63.ckpt",
".. /input/best-submission/model_epoch73.ckpt",
".. /input/best-submission/model_epoch89.ckpt",
".. /input/best-submission/model_epoch95.ckpt",
".. /input/best-submission/model_epoch99.ckpt"
]
weights = np.array([1,1,1,2,2,3,3,3,4,4,4,4,9,9,9,27,27,27], dtype=np.float32)
 weights /= weights.sum()
print(weights)
 nets = [ load_model(path)for path in paths]
assert len(paths)== len(weights)== len(nets)
 nets[0]<define_variables>
y_true = train[target_cols].values
y_pred = oof
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred.iloc[:, i],labels=[0,1])
 score +=(score_ / len(target_cols))
print("CV log_loss: ", score)
def make_colnames() :
cols= list(
 it.chain(*[
[f"coord_x0{i}", f"coord_y0{i}", f"coord_x1{i}", f"coord_y1{i}", f"coord_x2{i}", f"coord_y2{i}"]
for i in range(50)]))
return ["timestamp", "track_id"] + ["conf_0", "conf_1", "conf_2"] + cols<categorify>
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 y_true = train[target_cols].values
y_pred = train[target_cols].copy()
y_pred[target_cols] = 0
y_pred.loc[oof.index] = oof
y_pred.loc[train.cp_type=='ctl_vehicle'] = 0
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred.iloc[:, i],labels=[0,1])
 score +=(score_ / len(target_cols))
print("CV log_loss: ", score)
<predict_on_test><EOS> 
sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission.set_index('sig_id',inplace=True)
 test_features.set_index('sig_id',inplace=True)
 test_features = test_features.loc[sample_submission.index]
sub = sample_submission.drop(columns=target_cols ).merge(test[target_cols], on='sig_id', how='left' ).fillna(0)
 sub.loc[test_features['cp_type']=='ctl_vehicle', target_cols] =0
sub.to_csv('./submission.csv', index=True )